diff --git a/gateway/__init__.py b/gateway/__init__.py index 140cc32fc5..120327026b 100644 --- a/gateway/__init__.py +++ b/gateway/__init__.py @@ -1,13 +1,5 @@ -""" -Hermes Gateway - Multi-platform messaging integration. - -This module provides a unified gateway for connecting the Hermes agent -to various messaging platforms (Telegram, Discord, WhatsApp, Weixin, and more) with: -- Session management (persistent conversations with reset policies) -- Dynamic context injection (agent knows where messages come from) -- Delivery routing (cron job outputs to appropriate channels) -- Platform-specific toolsets (different capabilities per platform) -""" +"""Hermes Gateway - multi-platform messaging integration (sessions, context +injection, delivery routing, platform-specific toolsets).""" from .config import GatewayConfig, PlatformConfig, HomeChannel, load_gateway_config from .session import ( @@ -19,17 +11,7 @@ from .session import ( from .delivery import DeliveryRouter, DeliveryTarget __all__ = [ - # Config - "GatewayConfig", - "PlatformConfig", - "HomeChannel", - "load_gateway_config", - # Session - "SessionContext", - "SessionStore", - "SessionResetPolicy", - "build_session_context_prompt", - # Delivery - "DeliveryRouter", - "DeliveryTarget", + "GatewayConfig", "PlatformConfig", "HomeChannel", "load_gateway_config", + "SessionContext", "SessionStore", "SessionResetPolicy", "build_session_context_prompt", + "DeliveryRouter", "DeliveryTarget", ] diff --git a/gateway/agent_cache_pressure.py b/gateway/agent_cache_pressure.py index 1fc62dbadb..bf1270128d 100644 --- a/gateway/agent_cache_pressure.py +++ b/gateway/agent_cache_pressure.py @@ -1,15 +1,11 @@ """Memory-pressure bounds for the gateway's per-session AIAgent cache. -Each cached ``AIAgent`` pins ``_session_messages`` (the full live transcript, -tool outputs included — tens of MB on a tool-heavy session). The cache's LRU -cap counts entries, not bytes, and the idle TTL defers eviction for busy -sessions, so neither sees actual memory use. This module supplies that signal: -the process's own anonymous RSS against a budget derived from the cgroup limit. -``GatewayRunner._sweep_agent_cache_under_pressure`` uses it to shed LRU -transcripts via the soft-eviction path (rebuilt from the persisted session on -the next turn). - -Everything here is pure or read-only. Config lives under ``agent.agent_cache``. +Each cached ``AIAgent`` pins its full live transcript (tens of MB on a tool-heavy +session); the LRU cap counts entries, not bytes, and the idle TTL defers eviction +for busy sessions, so neither sees actual memory use. This module supplies that +signal — own anonymous RSS against a budget derived from the cgroup limit — and +``GatewayRunner`` sheds LRU transcripts via soft eviction (rebuilt from the +persisted session next turn). Pure/read-only; config under ``agent.agent_cache``. """ from __future__ import annotations @@ -20,29 +16,23 @@ from dataclasses import dataclass from pathlib import Path from typing import Any, Callable, Iterable, List, Optional, Tuple -# Shed well under the limit: once cgroup ``memory.high`` throttling kicks in -# (swap full), a SIGTERM flush cannot finish inside systemd's stop timeout. +# Shed well under the limit: once cgroup ``memory.high`` throttling kicks in (swap full), +# a SIGTERM flush cannot finish inside systemd's stop timeout. _AUTO_BUDGET_FRACTION = 0.65 -# Below this a budget is noise — small containers would evict every pass and -# never keep a warm prefix. +# Below this a budget is noise — small containers would evict every pass and never keep a warm prefix. _AUTO_BUDGET_FLOOR_MB = 512 - _DEFAULT_MAX_EVICTIONS_PER_PASS = 16 -# Never shed the hottest sessions: their prompt cache is worth the most, and -# evicting them just moves the cost to the next turn. +# Never shed the hottest sessions: their prompt cache is worth the most; evicting them +# just moves the cost to the next turn. _DEFAULT_PROTECT_RECENT = 8 - _BYTES_PER_MB = 1024 * 1024 +_OFF_WORDS = frozenset({"", "off", "none", "false", "disabled"}) @dataclass(frozen=True) class AgentCacheBounds: - """Operator-facing bounds for the per-session agent cache. - - ``max_size`` / ``idle_ttl_secs`` are ``None`` when unset so ``gateway/run.py`` - keeps its module defaults; ``memory_high_mb`` is ``None`` when pressure - eviction is off. - """ + """Operator-facing bounds. ``max_size``/``idle_ttl_secs`` are ``None`` when unset + so ``gateway/run.py`` keeps its defaults; ``memory_high_mb`` ``None`` = pressure eviction off.""" max_size: Optional[int] = None idle_ttl_secs: Optional[float] = None @@ -51,64 +41,43 @@ class AgentCacheBounds: protect_recent: int = _DEFAULT_PROTECT_RECENT -def _positive(value: Any, cast: Callable[[Any], Any]) -> Any: +def _is_int(value: Any) -> bool: + return isinstance(value, int) and not isinstance(value, bool) + + +def _positive(value: Any, cast: Callable[[Any], Any] = int) -> Any: """``cast(value)`` if it is a positive number (bools rejected), else None.""" - if isinstance(value, bool) or value is None: - return None try: - parsed = cast(value) + parsed = None if isinstance(value, bool) or value is None else cast(value) except (TypeError, ValueError): return None - return parsed if parsed > 0 else None - - -def _positive_int(value: Any) -> Optional[int]: - return _positive(value, int) - - -def _positive_float(value: Any) -> Optional[float]: - return _positive(value, float) + return parsed if parsed is not None and parsed > 0 else None def _cgroup_limit_bytes() -> Optional[int]: - """Return the memory limit this process runs under, if cgroup-capped. + """Memory limit this process runs under, if cgroup-capped. - Prefers cgroup v2 ``memory.high`` (the throttling point) over ``memory.max``, - then cgroup v1. Checks the process's *own* cgroup first (where a systemd - unit's ``MemoryHigh=``/``MemoryMax=`` lands — the root files read ``max`` - there), then the root for container-style limits. ``max`` and the v1 - near-2^63 sentinel mean unlimited. + Prefers v2 ``memory.high`` (the throttling point) over ``memory.max``, then v1. + Own cgroup first (where a systemd unit's ``MemoryHigh=``/``MemoryMax=`` lands — + root reads ``max`` there), then root for container-style limits. ``max`` and + the v1 near-2^63 sentinel mean unlimited. """ if sys.platform != "linux": return None - candidates: list[str] = [] try: from gateway.cgroup_cleanup import _own_cgroup_path own = _own_cgroup_path() except Exception: own = None - if own and own != "/": - candidates += [f"/sys/fs/cgroup{own}/memory.high", f"/sys/fs/cgroup{own}/memory.max"] - candidates += [ - "/sys/fs/cgroup/memory.high", - "/sys/fs/cgroup/memory.max", - "/sys/fs/cgroup/memory/memory.limit_in_bytes", - ] - for candidate in candidates: + roots = ([f"/sys/fs/cgroup{own}"] if own and own != "/" else []) + ["/sys/fs/cgroup"] + for candidate in [f"{r}/memory.{f}" for r in roots for f in ("high", "max")] + ["/sys/fs/cgroup/memory/memory.limit_in_bytes"]: try: - raw = Path(candidate).read_text(encoding="utf-8").strip() - except OSError: + limit = int(Path(candidate).read_text(encoding="utf-8").strip()) + except (OSError, ValueError): # unreadable, empty, or "max" continue - if not raw or raw == "max": - continue - try: - limit = int(raw) - except ValueError: - continue - if limit <= 0 or limit >= (1 << 62): - continue - return limit + if 0 < limit < (1 << 62): + return limit return None @@ -126,25 +95,16 @@ def _total_memory_bytes() -> Optional[int]: def resolve_memory_high_mb(setting: Any) -> Optional[int]: - """Resolve ``memory_high_mb`` into an absolute MB budget. - - ``"auto"`` derives it from the cgroup limit (or total RAM when uncapped); - a positive number is literal; anything falsy/off disables the pass. - """ + """Absolute MB budget: ``"auto"`` derives from the cgroup limit (or total RAM when + uncapped); a positive number is literal; anything falsy/off disables the pass.""" if isinstance(setting, str): normalized = setting.strip().lower() if normalized != "auto": - return ( - None - if normalized in ("", "off", "none", "false", "disabled") - else _positive_int(normalized) - ) - elif isinstance(setting, bool): - if not setting: - return None - else: - return _positive_int(setting) - + return None if normalized in _OFF_WORDS else _positive(normalized) + elif setting is False: + return None + elif setting is not True: + return _positive(setting) limit = _cgroup_limit_bytes() or _total_memory_bytes() if not limit: return None @@ -153,53 +113,29 @@ def resolve_memory_high_mb(setting: Any) -> Optional[int]: def resolve_agent_cache_bounds(config: Any) -> AgentCacheBounds: - """Read ``agent.agent_cache`` out of the *raw* config mapping. - - The gateway's loader does not deep-merge ``DEFAULT_CONFIG``, so an absent - key stays absent and callers can tell "operator chose 128" from "unset". - """ - section: Any = None - if isinstance(config, dict): - agent_cfg = config.get("agent") - if isinstance(agent_cfg, dict): - section = agent_cfg.get("agent_cache") + """Read ``agent.agent_cache`` from the *raw* config: the gateway loader does not + deep-merge ``DEFAULT_CONFIG``, so callers can tell "operator chose 128" from "unset".""" + section = (config.get("agent") or {}).get("agent_cache") if isinstance(config, dict) else None if not isinstance(section, dict): section = {} - - max_evictions = _positive_int(section.get("max_evictions_per_pass")) protect_recent = section.get("protect_recent") - protect_parsed = _positive_int(protect_recent) - # 0 means "shed anything" — distinct from unset. The bool guard keeps - # `protect_recent: false` (False == 0) on the default instead of silently - # disabling MRU protection. - if ( - protect_parsed is None - and isinstance(protect_recent, int) - and not isinstance(protect_recent, bool) - and protect_recent == 0 - ): + protect_parsed = _positive(protect_recent) + # 0 means "shed anything" — distinct from unset. The bool guard keeps `protect_recent: false` + # (False == 0) on the default instead of silently disabling MRU protection. + if protect_parsed is None and _is_int(protect_recent) and protect_recent == 0: protect_parsed = 0 - return AgentCacheBounds( - max_size=_positive_int(section.get("max_size")), - idle_ttl_secs=_positive_float(section.get("idle_ttl_secs")), + max_size=_positive(section.get("max_size")), + idle_ttl_secs=_positive(section.get("idle_ttl_secs"), float), memory_high_mb=resolve_memory_high_mb(section.get("memory_high_mb", "auto")), - max_evictions_per_pass=( - max_evictions if max_evictions is not None else _DEFAULT_MAX_EVICTIONS_PER_PASS - ), - protect_recent=( - protect_parsed if protect_parsed is not None else _DEFAULT_PROTECT_RECENT - ), + max_evictions_per_pass=_positive(section.get("max_evictions_per_pass")) or _DEFAULT_MAX_EVICTIONS_PER_PASS, + protect_recent=_DEFAULT_PROTECT_RECENT if protect_parsed is None else protect_parsed, ) def read_anon_rss_mb() -> Optional[int]: - """Return the process's anonymous resident memory in MB, or None. - - Anonymous pages are where cached transcripts live; file-backed pages are - noise. ``collect_memory_snapshot`` reads ``/proc/self/status`` without a - dependency; psutil covers other platforms (total RSS only). - """ + """Anonymous RSS in MB (where cached transcripts live; file-backed pages are noise), + or None. ``/proc/self/status`` first; psutil covers other platforms (total RSS only).""" try: from hermes_cli.mem_trim import collect_memory_snapshot @@ -210,7 +146,6 @@ def read_anon_rss_mb() -> Optional[int]: return kib // 1024 except Exception: pass - try: import psutil # type: ignore @@ -220,37 +155,27 @@ def read_anon_rss_mb() -> Optional[int]: def transcript_persistence_caught_up(agent: Any) -> bool: - """True when the agent's live transcript is fully on disk. + """True when the live transcript is fully on disk. - Soft eviction drops ``_session_messages`` and rebuilds from the persisted - session, so it is only safe once ``_last_flushed_db_idx`` (advanced to - ``len(messages)`` by ``AIAgent._flush_messages_to_session_db`` only on a - fully successful write) has caught up. Unknown shapes are *not* caught up: - a skipped eviction costs memory, a wrong one costs the conversation. + Soft eviction rebuilds from the persisted session, so it is only safe once + ``_last_flushed_db_idx`` (advanced only on a fully successful write) has caught + up. Unknown shapes are *not* caught up: a skipped eviction costs memory, a + wrong one costs the conversation. """ - messages = getattr(agent, "_session_messages", None) - if not isinstance(messages, list): - return False - flushed = getattr(agent, "_last_flushed_db_idx", None) - if not isinstance(flushed, int) or isinstance(flushed, bool): - return False - return flushed >= len(messages) + messages, flushed = getattr(agent, "_session_messages", None), getattr(agent, "_last_flushed_db_idx", None) + return isinstance(messages, list) and _is_int(flushed) and flushed >= len(messages) def plan_pressure_evictions( - ordered_entries: Iterable[Tuple[str, Any]], - *, - is_evictable: Callable[[str, Any], bool], - max_evictions: int, - protect_recent: int = 0, + ordered_entries: Iterable[Tuple[str, Any]], *, is_evictable: Callable[[str, Any], bool], + max_evictions: int, protect_recent: int = 0, ) -> List[Tuple[str, Any]]: """Choose which cached sessions to shed, least-recently-used first. - ``ordered_entries`` must be LRU→MRU (the cache OrderedDict is kept that way - by ``move_to_end`` on every hit). The batch is capped so one pass cannot - stall the gateway. ``protect_recent`` is clamped to half the cache: a few - huge transcripts can exhaust the budget alone, and a fixed guard would then - protect the whole cache with nothing left to shed. + ``ordered_entries`` must be LRU→MRU (the cache OrderedDict ``move_to_end``s on + every hit). The batch is capped so one pass cannot stall the gateway. + ``protect_recent`` is clamped to half the cache: a few huge transcripts can + exhaust the budget alone, and a fixed guard would leave nothing to shed. """ entries = list(ordered_entries) if max_evictions <= 0 or not entries: @@ -258,7 +183,6 @@ def plan_pressure_evictions( protect = min(max(protect_recent, 0), len(entries) // 2) if protect: entries = entries[:-protect] - plan: List[Tuple[str, Any]] = [] for key, agent in entries: if len(plan) >= max_evictions: diff --git a/gateway/authz_mixin.py b/gateway/authz_mixin.py index 04094779a0..61e15e257b 100644 --- a/gateway/authz_mixin.py +++ b/gateway/authz_mixin.py @@ -1,15 +1,13 @@ -"""User-authorization methods for ``GatewayRunner``. - -Inbound-message authorization cluster: whether a user/chat may talk to the +"""User-authorization mixin for ``GatewayRunner``: may this user/chat talk to the agent, the per-adapter DM policy, and the unauthorized-DM behavior. -``self.*`` calls resolve via the MRO. ``gateway.run`` is never imported at -module import time (cycle); the one method that logs imports its ``logger`` -lazily so records keep the ``"gateway.run"`` logger name. +``gateway.run`` is never imported at module import time (cycle); the one method +that logs imports its ``logger`` lazily so records keep the ``"gateway.run"`` name. """ from __future__ import annotations +import contextlib import os from typing import Optional @@ -22,6 +20,7 @@ from gateway.whatsapp_identity import ( ) _GROUP_CHAT_TYPES = frozenset({"group", "forum", "channel"}) +_GROUP_FORUM_TYPES = frozenset({"group", "forum"}) _TRUTHY = frozenset({"true", "1", "yes"}) # Platform -> ``_ALLOWED_USERS`` / ``_ALLOW_ALL_USERS``. @@ -47,51 +46,39 @@ _ALLOW_BOTS_ENV = { def _platform_gate_env(name: str, default: str = "") -> str: """Read an allow/deny gate env var with per-profile isolation. - When a profile secret scope is installed AND multiplexing is active, a - scoped miss returns ``default`` instead of falling through to - ``os.environ``: the process env may hold ANOTHER profile's first-writer - bridged value, so a fallthrough would leak allowlists across profiles. + With a profile secret scope installed AND multiplexing active, a scoped miss + returns ``default`` instead of falling through to ``os.environ``, which may + hold ANOTHER profile's first-writer bridged value (allowlist leak). Single-profile deployments behave exactly like ``os.getenv``. """ if not name: return default - try: + with contextlib.suppress(Exception): from agent.secret_scope import current_secret_scope, is_multiplex_active scope = current_secret_scope() if scope is not None and is_multiplex_active(): val = scope.get(name) return default if val is None else str(val).strip() - except Exception: - pass return (os.getenv(name) or default).strip() _auth_env = _platform_gate_env +def _env_truthy(name: str) -> bool: + return _auth_env(name).lower() in _TRUTHY + + def _registry_entry(platform): """Platform-registry entry for a (plugin) platform, or None.""" if platform is None: return None - try: + with contextlib.suppress(Exception): from gateway.platform_registry import platform_registry return platform_registry.get(platform.value) - except Exception: - return None - - -def _platform_declares_allowed_users_env(platform) -> bool: - """Whether a plugin platform's registry entry declares ``allowed_users_env``. - - Such platforms (Buzz, DingTalk, ...) document ``PlatformConfig.extra - .allowed_users`` as the config-file spelling of that env allowlist, so the - live adapter's extra is a valid authorization source when the env var is - absent. Built-in platforms and unknown entries return False. - """ - entry = _registry_entry(platform) - return bool(entry and entry.allowed_users_env) + return None def _coerce_allow_set(raw) -> set[str]: @@ -107,11 +94,12 @@ def _allows(allowed: set[str], candidate: Optional[str]) -> bool: return "*" in allowed or candidate in allowed -# --------------------------------------------------------------------------- -# Nostr npub -> hex (Buzz). ``BUZZ_ALLOWED_USERS`` accepts hex or ``npub1…`` -# but inbound pubkeys are always hex, so npub entries must be decoded. -# Pure stdlib; mirrors plugins/platforms/buzz/adapter.py. -# --------------------------------------------------------------------------- +def _adapter_config_extra(adapter) -> dict: + return getattr(getattr(adapter, "config", None), "extra", None) or {} + + +# Nostr npub -> hex (Buzz): ``BUZZ_ALLOWED_USERS`` accepts hex or ``npub1…`` but +# inbound pubkeys are always hex. Pure stdlib; mirrors plugins/platforms/buzz/adapter.py. _BECH32_CHARSET = "qpzry9x8gf2tvdw0s3jn54khce6mua7l" @@ -180,25 +168,50 @@ def _normalize_nostr_allow_entries(entries: set) -> set: return expanded +def _principal_matches_allowlist(source, user_id: str, allowed_ids: set) -> bool: + """Whether *user_id* (under any platform-specific alias) is in *allowed_ids*.""" + check_ids = {user_id} + if "@" in user_id: + check_ids.add(user_id.split("@")[0]) + + # WhatsApp (Baileys + Cloud): phone<->LID / JID aliases match the same principal. + if source.platform in {Platform.WHATSAPP, Platform.WHATSAPP_CLOUD}: + allowed_ids = set().union(*(_expand_whatsapp_auth_aliases(a) for a in allowed_ids)) or allowed_ids + check_ids.update(_expand_whatsapp_auth_aliases(user_id)) + normalized_user_id = _normalize_whatsapp_identifier(user_id) + if normalized_user_id: + check_ids.add(normalized_user_id) + + platform_value = source.platform.value if source.platform is not None else None + # SimpleX: user_id is the numeric contactId but the UI only shows display names. + if platform_value == "simplex" and source.user_name: + check_ids.add(source.user_name) + # Buzz: allowlist may hold npub or hex; inbound pubkeys are hex. + if platform_value == "buzz": + allowed_ids = _normalize_nostr_allow_entries(allowed_ids) + hex_user = _npub_to_hex(user_id) if user_id.startswith("npub") else None + if hex_user: + check_ids.add(hex_user) + return bool(check_ids & allowed_ids) + + class GatewayAuthorizationMixin: """User/chat authorization methods for ``GatewayRunner``.""" # ``getattr(self, ...)`` throughout: test helpers build bare runners via - # ``object.__new__`` without ``adapters`` / ``config`` (pitfalls.md #17). + # ``object.__new__`` without ``adapters`` / ``config``. - def _authorization_adapter( - self, - platform: Optional[Platform], - profile: Optional[str] = None, - ): - """Resolve the live adapter whose intake policy should gate authorization. + def _primary_adapters(self) -> dict: + return getattr(self, "adapters", None) or {} - Secondary-profile adapters live in ``_profile_adapters[profile]``; the - launched (primary) profile owns ``self.adapters``. ``_profile_adapters`` - is consulted BEFORE comparing against the active profile name: multiplex - turns override ``HERMES_HOME`` so ``_active_profile_name()`` reports the - secondary profile mid-turn, and treating it as primary would hand it the - default bot (or default-deny secondary-only platforms like A2A). + def _authorization_adapter(self, platform: Optional[Platform], profile: Optional[str] = None): + """Live adapter whose intake policy gates authorization. + + Secondary-profile adapters live in ``_profile_adapters[profile]``; the primary + profile owns ``self.adapters``. ``_profile_adapters`` is consulted BEFORE the + active profile name: multiplex turns override ``HERMES_HOME`` so + ``_active_profile_name()`` reports the secondary profile mid-turn, and treating + it as primary would hand it the default bot. """ if not platform: return None @@ -207,23 +220,17 @@ class GatewayAuthorizationMixin: profile_adapters = getattr(self, "_profile_adapters", None) or {} if profile_name in profile_adapters: return profile_adapters[profile_name].get(platform) - # Compare against the identity captured at construction, not the - # per-turn HERMES_HOME-derived name. + # Identity captured at construction, not the per-turn HERMES_HOME-derived name. primary_profile = getattr(self, "_primary_profile_name", None) if not primary_profile: - active_profile_fn = getattr(self, "_active_profile_name", None) - if callable(active_profile_fn): - try: - primary_profile = active_profile_fn() - except Exception: - primary_profile = None + with contextlib.suppress(Exception): + primary_profile = self._active_profile_name() if profile_name == primary_profile: - return (getattr(self, "adapters", None) or {}).get(platform) - # Fail closed: a stamped secondary profile with no registry entry - # (adapter failed to connect) must NOT fall back to the default - # profile's adapter -- that sends replies out the wrong bot. + return self._primary_adapters().get(platform) + # Fail closed: a secondary profile whose adapter failed to connect must NOT + # fall back to the default profile's adapter (replies out the wrong bot). return None - return (getattr(self, "adapters", None) or {}).get(platform) + return self._primary_adapters().get(platform) def _adapter_for_source(self, source: Optional[SessionSource]): """Resolve the live adapter for an inbound ``SessionSource``.""" @@ -232,22 +239,17 @@ class GatewayAuthorizationMixin: transport_adapter = self._registered_transport_adapter(source) if transport_adapter is not None: return transport_adapter - # Relay ingress keeps the underlying platform on the source (session - # keys / display policy), but delivery must use the one process-level - # RelayAdapter that owns the connector socket -- secondary profiles do - # not register their own, so a profile-aware lookup would silently - # disable streaming/typing/tool progress. + # Relay ingress keeps the underlying platform on the source, but delivery must + # use the one process-level RelayAdapter owning the connector socket; a + # profile-aware lookup would silently disable streaming/typing/tool progress. if getattr(source, "delivered_via_upstream_relay", False) is True: - return (getattr(self, "adapters", None) or {}).get(Platform.RELAY) + return self._primary_adapters().get(Platform.RELAY) # ``getattr``: test fixtures build bare SimpleNamespace sources without ``profile``. - return self._authorization_adapter( - getattr(source, "platform", None), - getattr(source, "profile", None), - ) + return self._authorization_adapter(getattr(source, "platform", None), getattr(source, "profile", None)) def _owning_profile(self, adapter, platform): """Return (registered, profile) for a live adapter: profile is None for primary.""" - if adapter is (getattr(self, "adapters", None) or {}).get(platform): + if adapter is self._primary_adapters().get(platform): return True, None for profile, profile_adapters in (getattr(self, "_profile_adapters", None) or {}).items(): if adapter is profile_adapters.get(platform): @@ -255,13 +257,12 @@ class GatewayAuthorizationMixin: return False, None def _registered_transport_adapter(self, source: SessionSource): - """Return the registered adapter that created *source*, if retained. + """The registered adapter that created *source*, if retained. - ``source.profile`` is the runtime/session namespace and may differ from - the adapter profile when one shared credential serves several routed - runtimes; ``build_source`` keeps the receiving adapter as in-process - provenance so replies and intake-policy checks stay on that transport. - Restored or hand-built sources fall back (fail-closed) to profile lookup. + ``source.profile`` may differ from the adapter profile when one shared credential + serves several routed runtimes; ``build_source`` keeps the receiving adapter as + provenance so replies stay on that transport. Restored/hand-built sources fall + back (fail-closed) to profile lookup. """ adapter_ref = getattr(source, "_transport_adapter_ref", None) adapter = adapter_ref() if callable(adapter_ref) else None @@ -286,102 +287,57 @@ class GatewayAuthorizationMixin: adapter = self._authorization_adapter(platform, profile) return adapter is not None and bool(getattr(adapter, name, False)) - def _adapter_authorization_is_upstream( - self, - platform: Optional[Platform], - *, - profile: Optional[str] = None, - ) -> bool: - """Whether the adapter delegates authz to a trusted authenticated upstream. + def _adapter_authorization_is_upstream(self, platform: Optional[Platform], *, profile: Optional[str] = None) -> bool: + """Whether the adapter delegates authz to a trusted authenticated upstream (relay). - Mirrors ``BasePlatformAdapter.authorization_is_upstream`` (True for the - relay adapter: the connector authenticates the WS and resolves owner - bindings before delivering). Unlike ``_adapter_enforces_own_access_policy`` - (a LOCAL policy only trusted when it is an allowlist) this UPSTREAM - decision is honored directly. False when the adapter is unknown. + Unlike ``_adapter_enforces_own_access_policy`` (a LOCAL policy only trusted when + it is an allowlist) this UPSTREAM decision is honored directly. False when unknown. """ return self._adapter_flag(platform, "authorization_is_upstream", profile) - def _adapter_enforces_own_access_policy( - self, - platform: Optional[Platform], - *, - profile: Optional[str] = None, - ) -> bool: - """Whether the adapter gates access at intake itself. + def _adapter_enforces_own_access_policy(self, platform: Optional[Platform], *, profile: Optional[str] = None) -> bool: + """Whether the adapter gates access at intake itself (WeCom, Weixin, Yuanbao, QQBot, WhatsApp). - Mirrors ``BasePlatformAdapter.enforces_own_access_policy`` (WeCom, Weixin, - Yuanbao, QQBot, WhatsApp). The flag alone is NOT "already authorized": - these adapters default to ``open``, so ``_is_user_authorized`` only trusts - them when the effective policy is an actual ``allowlist``. + The flag alone is NOT "already authorized": these adapters default to ``open``, + so ``_is_user_authorized`` only trusts them under an actual ``allowlist`` policy. """ return self._adapter_flag(platform, "enforces_own_access_policy", profile) def _config_extra(self, platform) -> dict: """``config.platforms[platform].extra`` as a dict ({} when absent).""" - config = getattr(self, "config", None) - platform_cfg = ( - config.platforms.get(platform) - if config is not None and hasattr(config, "platforms") - else None - ) - extra = getattr(platform_cfg, "extra", None) if platform_cfg else None + platforms = getattr(getattr(self, "config", None), "platforms", None) + extra = getattr(platforms.get(platform), "extra", None) if platforms is not None else None return extra if isinstance(extra, dict) else {} def _adapter_setting(self, platform, attr: str, extra_key: str, profile): - """Live adapter's resolved ``attr``, else ``config.extra[extra_key]``. - - The adapter value is preferred because it already folds in the - ``_*`` env override, which is not always bridged back into - ``config.extra``; the extra fallback serves bare runners with no adapter. - """ + """Live adapter's resolved ``attr`` (folds in the ``_*`` env override), + else ``config.extra[extra_key]`` for bare runners with no adapter.""" adapter = self._authorization_adapter(platform, profile) value = getattr(adapter, attr, None) if adapter is not None else None if value is None: value = self._config_extra(platform).get(extra_key) return value - def _adapter_dm_policy( - self, - platform: Optional[Platform], - *, - profile: Optional[str] = None, - ) -> str: - """Lowercased effective ``dm_policy`` (open/allowlist/disabled/pairing), ``""`` if unknown. - - "Reached the gateway" only carries an authorization signal in the - ``allowlist`` case; ``open`` forwards everyone and ``pairing`` forwards - unpaired DMs for the handshake. - """ + def _adapter_policy(self, platform, attr: str, extra_key: str, profile) -> str: if not platform: return "" - policy = self._adapter_setting(platform, "_dm_policy", "dm_policy", profile) - return str(policy or "").strip().lower() + return str(self._adapter_setting(platform, attr, extra_key, profile) or "").strip().lower() - def _adapter_group_policy( - self, - platform: Optional[Platform], - *, - profile: Optional[str] = None, - ) -> str: + def _adapter_dm_policy(self, platform: Optional[Platform], *, profile: Optional[str] = None) -> str: + """Lowercased effective ``dm_policy`` (open/allowlist/disabled/pairing), ``""`` if unknown.""" + return self._adapter_policy(platform, "_dm_policy", "dm_policy", profile) + + def _adapter_group_policy(self, platform: Optional[Platform], *, profile: Optional[str] = None) -> str: """Lowercased effective ``group_policy`` (open/allowlist/disabled), ``""`` if unknown.""" - if not platform: - return "" - policy = self._adapter_setting(platform, "_group_policy", "group_policy", profile) - return str(policy or "").strip().lower() + return self._adapter_policy(platform, "_group_policy", "group_policy", profile) def _adapter_group_has_sender_allowlist( - self, - platform: Optional[Platform], - chat_id: Optional[str], - *, - profile: Optional[str] = None, + self, platform: Optional[Platform], chat_id: Optional[str], *, profile: Optional[str] = None ) -> bool: """Whether a per-group sender allowlist (WeCom ``groups..allow_from``) gated this message. - A group may be open at the chat level while restricting senders; if such - a message reached the gateway the adapter already checked that sender - allowlist, so it is a trustworthy intake decision. + A group may be open at the chat level while restricting senders; reaching the + gateway then means the adapter already checked that list. """ if not platform or not chat_id: return False @@ -393,12 +349,10 @@ class GatewayAuthorizationMixin: group_cfg = groups.get(chat_id_str) if not isinstance(group_cfg, dict): lowered = chat_id_str.lower() - for key, value in groups.items(): - if isinstance(key, str) and key.lower() == lowered and isinstance(value, dict): - group_cfg = value - break - if not isinstance(group_cfg, dict): - group_cfg = groups.get("*") + group_cfg = next( + (v for k, v in groups.items() if isinstance(k, str) and k.lower() == lowered and isinstance(v, dict)), + groups.get("*"), + ) if not isinstance(group_cfg, dict): return False @@ -418,43 +372,100 @@ class GatewayAuthorizationMixin: return getattr(self, "pairing_store", None) def _adapter_extra_for_source(self, source) -> dict: + return _adapter_config_extra(self._adapter_for_source(source)) + + def _own_policy_authorizes(self, source, user_id, is_group, adapter_profile) -> Optional[bool]: + """Own-policy adapter verdict when no env allowlist exists; None = no verdict. + + Trusted only when the effective policy for THIS chat type is ``allowlist``: + ``open`` forwards EVERY sender (the fail-open SECURITY.md §2.6 forbids), + ``disabled`` never forwards, ``pairing`` forwards unpaired DMs for the handshake + (already denied by the pairing-store check). Anything else → default-deny. + """ + if is_group: + if self._adapter_group_has_sender_allowlist(source.platform, source.chat_id, profile=adapter_profile): + return True + effective_policy = self._adapter_group_policy(source.platform, profile=adapter_profile) + else: + effective_policy = self._adapter_dm_policy(source.platform, profile=adapter_profile) + if effective_policy != "allowlist": + return None + # Re-check DMs via the live adapter's ``_is_dm_allowed`` when present: pairing + # revoke can clear WHATSAPP_ALLOWED_USERS while a construction-time snapshot + # would keep authorizing until restart. Others keep the historical rubber-stamp. + if not is_group: + adapter = self._authorization_adapter(source.platform, profile=adapter_profile) + dm_check = getattr(adapter, "_is_dm_allowed", None) if adapter is not None else None + if callable(dm_check): + return bool(dm_check(user_id)) + return True + + def _adapter_extra_allowlist_authorizes(self, source, user_id, is_group) -> bool: + """Adapters (e.g. Telegram) that gate via config.extra.allow_from / group_allow_from + without setting enforces_own_access_policy.""" adapter = self._adapter_for_source(source) if adapter is None: - return {} - return getattr(getattr(adapter, "config", None), "extra", None) or {} + return False + extra = _adapter_config_extra(adapter) + adapter_allow = extra.get("group_allow_from" if is_group else "allow_from") + if not adapter_allow: + # Plugin platforms (Buzz, DingTalk) spell their env allowlist as + # ``extra.allowed_users``; under multiplex only the default profile's list + # reaches the env (first-writer-wins bridge), so read the live adapter's. + entry = _registry_entry(source.platform) + if entry and entry.allowed_users_env: + adapter_allow = extra.get("allowed_users") + if not adapter_allow: + return False + allowed = _coerce_allow_set(adapter_allow) + normalize = getattr(adapter, "normalize_user_id", None) + if callable(normalize): + # Ids and entries may spell the same principal differently (Buzz hex vs npub). + allowed = {normalize(entry) or entry for entry in allowed} + return _allows(allowed, user_id) - def _is_user_authorized( - self, - source: SessionSource, - *, - allow_adapter_delegation: bool = True, - ) -> bool: - """Check if a user is authorized to use the bot. + def _adapter_resolved_allowlist_ids(self, source) -> set[str]: + """IDs an adapter resolved from username-shaped allowlist entries at connect time (Discord). + + The per-turn .env hot-reload restores RAW usernames, so from the second turn on the + env allowlist holds usernames while user_id is numeric. Never a widening: the + empty-allowlist branch already returned and adapters only resolve operator-written + entries. Only called with a non-empty platform allowlist so group/global-only + configs never consult adapter memory; type-checked so mocks cannot auto-truthy in. + """ + adapter = resolved_ids = None + with contextlib.suppress(Exception): + adapter = self._adapter_for_source(source) + resolver = getattr(adapter, "resolved_allowlist_user_ids", None) + if callable(resolver): + with contextlib.suppress(Exception): + resolved_ids = resolver() + if not isinstance(resolved_ids, (set, frozenset, list, tuple)): + return set() + return {str(entry).strip() for entry in resolved_ids if isinstance(entry, (str, int)) and str(entry).strip()} + + def _is_user_authorized(self, source: SessionSource, *, allow_adapter_delegation: bool = True) -> bool: + """Whether a user may use the bot. Order: trusted-upstream delegation, chat-scoped group allowlists, - ``{PLATFORM}_ALLOW_BOTS``, per-platform allow-all, adapter role auth, - pairing store, env/config allowlists, ``GATEWAY_ALLOW_ALL_USERS``, - default deny. + ``{PLATFORM}_ALLOW_BOTS``, per-platform allow-all, adapter role auth, pairing + store, env/config allowlists, ``GATEWAY_ALLOW_ALL_USERS``, default deny. """ from gateway.run import logger - # HA events are system-generated (HASS_TOKEN authenticates the - # connection); webhook events are HMAC-verified in the adapter. + # HA events are system-generated (HASS_TOKEN); webhook events are HMAC-verified. if source.platform in {Platform.HOMEASSISTANT, Platform.WEBHOOK}: return True adapter_profile = self._adapter_profile_for_source(source) is_group = source.chat_type in _GROUP_CHAT_TYPES + is_group_or_forum = source.chat_type in _GROUP_FORUM_TYPES - # Trusted-upstream delegation (relay): the Team Gateway connector - # authenticates this gateway's WS and resolves owner-only author - # bindings BEFORE delivering, so the event is already authorized and - # there is no local RELAY_ALLOWED_USERS to consult. Not a fail-open: it - # fires only for events actually delivered over the authenticated relay - # WS (transport stamps ``delivered_via_upstream_relay``) or whose - # adapter declares ``authorization_is_upstream=True``. The delivery - # marker is PRIMARY because a relayed message carries the UNDERLYING - # platform (discord/...), not ``Platform.RELAY``; the adapter-flag check - # covers events whose platform IS RELAY (interaction passthrough). + # Trusted-upstream delegation (relay): the connector authenticates this + # gateway's WS and resolves owner bindings BEFORE delivering, so there is no + # local RELAY_ALLOWED_USERS. Not a fail-open: fires only for events actually + # delivered over the relay WS (``delivered_via_upstream_relay``) or whose adapter + # declares ``authorization_is_upstream``. The delivery marker is PRIMARY because + # a relayed message carries the UNDERLYING platform, not ``Platform.RELAY``. # ``is True``: a MagicMock stand-in must not auto-truthy into authz. if allow_adapter_delegation and ( source.delivered_via_upstream_relay is True @@ -464,33 +475,21 @@ class GatewayAuthorizationMixin: user_id = source.user_id - # Chat-scoped group allowlists (TELEGRAM_GROUP_ALLOWED_CHATS / - # QQ_GROUP_ALLOWED_USERS) must work with ``user_id is None`` (anonymous - # admins, sender_chat posts, channel broadcasts), so they run before - # the no-user-id guard. + # Chat-scoped group allowlists must work with ``user_id is None`` (anonymous + # admins, sender_chat posts, channel broadcasts): run before the no-user-id guard. if is_group and source.chat_id: chat_allowlist_env = _GROUP_CHAT_ENV.get(source.platform, "") - if chat_allowlist_env and _allows( - _coerce_allow_set(_platform_gate_env(chat_allowlist_env)), source.chat_id - ): + if chat_allowlist_env and _allows(_coerce_allow_set(_platform_gate_env(chat_allowlist_env)), source.chat_id): return True - # config.yaml fallback (``extra.group_allowed_chats``): Telegram - # observe-unmentioned mode strips user_id from triggered group - # messages, so the env-only check above misses config allowlists. - try: - adapter_group_allowed = self._adapter_extra_for_source(source).get( - "group_allowed_chats" - ) - if adapter_group_allowed and _allows( - _coerce_allow_set(adapter_group_allowed), source.chat_id - ): + # config.yaml fallback (``extra.group_allowed_chats``): Telegram observe- + # unmentioned mode strips user_id, so the env-only check above misses it. + with contextlib.suppress(Exception): + adapter_group_allowed = self._adapter_extra_for_source(source).get("group_allowed_chats") + if adapter_group_allowed and _allows(_coerce_allow_set(adapter_group_allowed), source.chat_id): return True - except Exception: - pass - # Bots admitted by {PLATFORM}_ALLOW_BOTS bypass the human allowlist. - # Also before the no-user-id guard: Slack Workflow Builder posts arrive - # as bot_message with user=None. + # Bots admitted by {PLATFORM}_ALLOW_BOTS bypass the human allowlist. Also before + # the no-user-id guard: Slack Workflow Builder posts arrive with user=None. if getattr(source, "is_bot", False): allow_bots_var = _ALLOW_BOTS_ENV.get(source.platform) if allow_bots_var and _platform_gate_env(allow_bots_var, "none").lower().strip() in {"mentions", "all"}: @@ -499,120 +498,60 @@ class GatewayAuthorizationMixin: if not user_id: return False - platform_env_map = dict(_ALLOWED_USERS_ENV) - platform_allow_all_map = dict(_ALLOW_ALL_ENV) - if source.platform not in platform_env_map: - try: - entry = _registry_entry(source.platform) - if entry: - if entry.allowed_users_env: - platform_env_map[source.platform] = entry.allowed_users_env - if entry.allow_all_env: - platform_allow_all_map[source.platform] = entry.allow_all_env - except Exception: - pass + platform_allow_env = _ALLOWED_USERS_ENV.get(source.platform, "") + platform_allow_all_var = _ALLOW_ALL_ENV.get(source.platform, "") + if source.platform not in _ALLOWED_USERS_ENV: + entry = _registry_entry(source.platform) + with contextlib.suppress(Exception): + platform_allow_env = getattr(entry, "allowed_users_env", "") or platform_allow_env + platform_allow_all_var = getattr(entry, "allow_all_env", "") or platform_allow_all_var - platform_allow_all_var = platform_allow_all_map.get(source.platform, "") - if platform_allow_all_var and _auth_env(platform_allow_all_var).lower() in _TRUTHY: + if platform_allow_all_var and _env_truthy(platform_allow_all_var): return True - # Adapter-verified role auth (Discord confirmed DISCORD_ALLOWED_ROLES - # before dispatch). ``is True`` so a MagicMock source cannot pass. + # Adapter-verified role auth (Discord DISCORD_ALLOWED_ROLES). ``is True``: no MagicMock pass. if allow_adapter_delegation and getattr(source, "role_authorized", False) is True: return True - # Pairing store: a first-class grant created only by a trusted operator - # approving a code (inbound senders can never reach approve_code). - # Honored as a UNION with the allowlist; approval also mirrors the user - # into a configured allowlist (PairingStore._approve_user). + # Pairing store: a first-class grant created only by an operator approving a + # code. Honored as a UNION with the allowlist (approval also mirrors into it). platform_name = source.platform.value if source.platform else "" pairing_store = self._pairing_store_for(source) if pairing_store is not None and pairing_store.is_approved(platform_name, user_id): return True - platform_allowlist = _auth_env(platform_env_map.get(source.platform, "")) + platform_allowlist = _auth_env(platform_allow_env) group_user_allowlist = "" group_chat_allowlist = "" - if source.chat_type in {"group", "forum"}: + if is_group_or_forum: group_user_allowlist = _auth_env(_GROUP_USER_ENV.get(source.platform, "")) group_chat_allowlist = _auth_env(_GROUP_CHAT_ENV.get(source.platform, "")) global_allowlist = _auth_env("GATEWAY_ALLOWED_USERS") - if not platform_allowlist and not group_user_allowlist and not group_chat_allowlist and not global_allowlist: - # No env allowlist. Own-policy adapters gate at intake, but their - # decision is only trusted when the effective policy for THIS chat - # type is ``allowlist``: ``open`` forwards EVERY sender (trusting it - # is the fail-open SECURITY.md §2.6 forbids), ``disabled`` never - # forwards, ``pairing`` forwards unpaired DMs for the handshake - # (already denied by the pairing-store check above). Anything else - # falls through to default-deny; GATEWAY_ALLOW_ALL_USERS, the - # per-platform ALLOW_ALL flag and pairing stay the explicit opt-ins. + if not (platform_allowlist or group_user_allowlist or group_chat_allowlist or global_allowlist): + # No env allowlist: own-policy adapters gate at intake (see _own_policy_authorizes). if allow_adapter_delegation and self._adapter_enforces_own_access_policy( source.platform, profile=adapter_profile ): - if is_group: - effective_policy = self._adapter_group_policy(source.platform, profile=adapter_profile) - if self._adapter_group_has_sender_allowlist( - source.platform, source.chat_id, profile=adapter_profile - ): - return True - else: - effective_policy = self._adapter_dm_policy(source.platform, profile=adapter_profile) - if effective_policy == "allowlist": - # Re-check DMs against the live adapter when it exposes - # ``_is_dm_allowed``: pairing revoke can clear - # WHATSAPP_ALLOWED_USERS while a construction-time - # ``_allow_from`` snapshot would keep authorizing until - # restart. Adapters without the helper keep the historical - # "reached the gateway under allowlist policy" rubber-stamp. - if not is_group: - adapter = self._authorization_adapter(source.platform, profile=adapter_profile) - dm_check = getattr(adapter, "_is_dm_allowed", None) if adapter is not None else None - if callable(dm_check): - return bool(dm_check(user_id)) - return True - # Adapters (e.g. Telegram) that gate via config.extra.allow_from / - # group_allow_from without setting enforces_own_access_policy. - adapter = self._adapter_for_source(source) - if adapter is not None: - extra = getattr(getattr(adapter, "config", None), "extra", None) or {} - adapter_allow = extra.get("group_allow_from" if is_group else "allow_from") - if not adapter_allow and _platform_declares_allowed_users_env(source.platform): - # Plugin platforms (Buzz) carry the env allowlist as - # ``extra.allowed_users``. Under multiplex the YAML->env - # bridge is first-writer-wins, so only the default profile's - # list reaches the env read above; consult the live - # profile-routed adapter's config instead. - adapter_allow = extra.get("allowed_users") - if adapter_allow: - allowed = _coerce_allow_set(adapter_allow) - normalize = getattr(adapter, "normalize_user_id", None) - if callable(normalize): - # Ids and entries may spell the same principal - # differently (Buzz hex vs npub). - allowed = {normalize(entry) or entry for entry in allowed} - if _allows(allowed, user_id): - return True - return _auth_env("GATEWAY_ALLOW_ALL_USERS").lower() in _TRUTHY + verdict = self._own_policy_authorizes(source, user_id, is_group, adapter_profile) + if verdict is not None: + return verdict + if self._adapter_extra_allowlist_authorizes(source, user_id, is_group): + return True + return _env_truthy("GATEWAY_ALLOW_ALL_USERS") # Telegram group traffic authorized by chat ID (separate from # TELEGRAM_GROUP_ALLOWED_USERS, which gates the sender). - if group_chat_allowlist and source.chat_type in {"group", "forum"} and source.chat_id: - if _allows(_coerce_allow_set(group_chat_allowlist), source.chat_id): - return True - - # Backward-compat: TELEGRAM_GROUP_ALLOWED_USERS was once (mis)used as a - # chat-ID allowlist. "-"-prefixed values are chat IDs, so honor them as - # such and warn once; the correct var is TELEGRAM_GROUP_ALLOWED_CHATS. if ( - source.platform == Platform.TELEGRAM - and group_user_allowlist - and source.chat_type in {"group", "forum"} - and source.chat_id + group_chat_allowlist and is_group_or_forum and source.chat_id + and _allows(_coerce_allow_set(group_chat_allowlist), source.chat_id) ): - legacy_chat_ids = { - v.strip() for v in group_user_allowlist.split(",") if v.strip().startswith("-") - } + return True + + # Backward-compat: TELEGRAM_GROUP_ALLOWED_USERS was once (mis)used as a chat-ID + # allowlist; "-"-prefixed values are chat IDs, honor them and warn once. + if source.platform == Platform.TELEGRAM and group_user_allowlist and is_group_or_forum and source.chat_id: + legacy_chat_ids = {v.strip() for v in group_user_allowlist.split(",") if v.strip().startswith("-")} if legacy_chat_ids: if not getattr(self, "_warned_telegram_group_users_legacy", False): logger.warning( @@ -626,113 +565,39 @@ class GatewayAuthorizationMixin: if source.chat_id in legacy_chat_ids: return True - # In group/forum chats TELEGRAM_GROUP_ALLOWED_USERS is the scoped - # allowlist and does not imply DM access; TELEGRAM_ALLOWED_USERS is - # platform-wide and works everywhere. + # TELEGRAM_GROUP_ALLOWED_USERS is group-scoped (no DM access); TELEGRAM_ALLOWED_USERS is platform-wide. allowed_ids = ( _coerce_allow_set(platform_allowlist) | _coerce_allow_set(group_user_allowlist) | _coerce_allow_set(global_allowlist) ) - - # Adapters that resolve username-shaped entries to numeric IDs at - # connect time (Discord) keep the resolved set in memory; the per-turn - # .env hot-reload restores the RAW usernames into the env, so from the - # second turn on ``platform_allowlist`` holds usernames while user_id is - # numeric. Union in the adapter's resolved IDs. Never a widening: the - # empty-allowlist branch already returned and adapters only resolve - # operator-written entries. Guarded on ``platform_allowlist`` so - # group/global-only configs never consult adapter memory; duck-typed + - # type-checked so mock adapters cannot auto-truthy in. if platform_allowlist: - try: - adapter = self._adapter_for_source(source) - except Exception: - adapter = None - resolver = getattr(adapter, "resolved_allowlist_user_ids", None) - if callable(resolver): - try: - resolved_ids = resolver() - except Exception: - resolved_ids = None - if isinstance(resolved_ids, (set, frozenset, list, tuple)): - allowed_ids.update( - str(entry).strip() - for entry in resolved_ids - if isinstance(entry, (str, int)) and str(entry).strip() - ) + allowed_ids |= self._adapter_resolved_allowlist_ids(source) + return "*" in allowed_ids or _principal_matches_allowlist(source, user_id, allowed_ids) - if "*" in allowed_ids: - return True + def _get_unauthorized_dm_behavior(self, platform: Optional[Platform], *, profile: Optional[str] = None) -> str: + """How unauthorized DMs are handled ("pair" / "ignore") for a platform. - check_ids = {user_id} - if "@" in user_id: - check_ids.add(user_id.split("@")[0]) - - # WhatsApp (Baileys + Cloud): resolve phone<->LID / JID aliases so - # device-suffix and bare-phone entries match the same principal. - if source.platform in {Platform.WHATSAPP, Platform.WHATSAPP_CLOUD}: - normalized_allowed_ids = set() - for allowed_id in allowed_ids: - normalized_allowed_ids.update(_expand_whatsapp_auth_aliases(allowed_id)) - if normalized_allowed_ids: - allowed_ids = normalized_allowed_ids - - check_ids.update(_expand_whatsapp_auth_aliases(user_id)) - normalized_user_id = _normalize_whatsapp_identifier(user_id) - if normalized_user_id: - check_ids.add(normalized_user_id) - - platform_value = source.platform.value if source.platform is not None else None - - # SimpleX: user_id is the numeric contactId (stable across renames) but - # the UI only surfaces display names, so match both. Plugin platform: - # compare by value. - if platform_value == "simplex" and source.user_name: - check_ids.add(source.user_name) - - # Buzz: allowlist may hold npub or hex; inbound pubkeys are hex. - if platform_value == "buzz": - allowed_ids = _normalize_nostr_allow_entries(allowed_ids) - if user_id.startswith("npub"): - hex_user = _npub_to_hex(user_id) - if hex_user: - check_ids.add(hex_user) - - return bool(check_ids & allowed_ids) - - def _get_unauthorized_dm_behavior( - self, - platform: Optional[Platform], - *, - profile: Optional[str] = None, - ) -> str: - """Return how unauthorized DMs should be handled for a platform. - - Resolution order: - 1. Explicit per-platform ``unauthorized_dm_behavior`` in config. - 2. Email -> ``"ignore"`` (inboxes hold arbitrary human mail; pairing - codes are not a safe default; matches GatewayConfig). - 3. Explicit (non-default) global ``unauthorized_dm_behavior``. - 4. Adapter dm_policy: ``pairing`` -> ``"pair"``; ``allowlist`` / - ``disabled`` -> ``"ignore"``. - 5. Any configured allowlist -> ``"ignore"`` (the operator restricted - access; spamming unknown contacts with codes is noisy and leaks). - 6. Otherwise ``"pair"`` (open-gateway default). + Order: explicit per-platform config; Email → "ignore" (inboxes hold arbitrary + mail); explicit non-default global; adapter dm_policy (pairing → "pair", + allowlist/disabled → "ignore"); any configured allowlist → "ignore" (spamming + unknown contacts with codes is noisy and leaks); else "pair". """ config = getattr(self, "config", None) - if config and hasattr(config, "get_unauthorized_dm_behavior") and platform: - if "unauthorized_dm_behavior" in self._config_extra(platform): - return config.get_unauthorized_dm_behavior(platform) + if ( + config and hasattr(config, "get_unauthorized_dm_behavior") and platform + and "unauthorized_dm_behavior" in self._config_extra(platform) + ): + return config.get_unauthorized_dm_behavior(platform) if platform == Platform.EMAIL: return "ignore" - if config and hasattr(config, "unauthorized_dm_behavior"): - if config.unauthorized_dm_behavior != "pair": - return config.unauthorized_dm_behavior + if config and hasattr(config, "unauthorized_dm_behavior") and config.unauthorized_dm_behavior != "pair": + return config.unauthorized_dm_behavior + allowlist_keys = ["GATEWAY_ALLOWED_USERS"] if platform: dm_policy = self._adapter_dm_policy(platform, profile=profile) if not dm_policy: @@ -741,15 +606,9 @@ class GatewayAuthorizationMixin: return "pair" if dm_policy in {"allowlist", "disabled"}: return "ignore" - # Historical: Yuanbao is absent from this allowlist-aware default. env_key = "" if platform == Platform.YUANBAO else _ALLOWED_USERS_ENV.get(platform, "") - group_keys = (_GROUP_USER_ENV.get(platform), _GROUP_CHAT_ENV.get(platform)) - for key in (env_key, *group_keys): - if key and _platform_gate_env(key).strip(): - return "ignore" - - if _platform_gate_env("GATEWAY_ALLOWED_USERS").strip(): + allowlist_keys = [env_key, _GROUP_USER_ENV.get(platform), _GROUP_CHAT_ENV.get(platform), *allowlist_keys] + if any(key and _platform_gate_env(key).strip() for key in allowlist_keys): return "ignore" - return "pair" diff --git a/gateway/browser_control_artifacts.py b/gateway/browser_control_artifacts.py index 7b31875d07..732eb06c53 100644 --- a/gateway/browser_control_artifacts.py +++ b/gateway/browser_control_artifacts.py @@ -1,27 +1,9 @@ -"""One-shot artifact transport for browser control (Gateway side). - -Transport-neutral store core for bounded browser-control artifacts -(screenshots, PDFs, uploads). It knows nothing about aiohttp: the routes in -:mod:`gateway.platforms.api_server` authenticate callers and rate-limit, then -hand bytes here. The controller WebSocket is a command channel, not a file -pipe — frames carry only ``artifact_id`` strings and the bytes live on disk -under a controlled root for a short TTL. - -Contract (tests/gateway/test_browser_control_artifacts.py): - -- Server-minted ``[0-9a-f]{32}`` ids resolved strictly inside the root; - client filenames are metadata only, never paths. -- Exact size and MIME caps enforced before any disk write. -- SHA-256 recorded in the receipt and re-verified by ``load``/``validate``. -- ``load`` requires the exact scope key and consumes atomically; - ``validate`` checks existence/TTL/scope without consuming. -- No overwrite: an id collision is retried with a fresh id. -- ``prune_expired`` removes expired entries; the API server sweeps on demand. - -Thread-safety: the in-memory index is lock-guarded; files are written to a -temp name and atomically renamed so a concurrent ``load`` never sees a -partial artifact. -""" +"""One-shot artifact transport for browser control (Gateway side); :mod:`gateway.platforms.api_server` +authenticates and rate-limits, then hands bytes here. Controller frames carry only server-minted +``[0-9a-f]{32}`` ids (client filenames are metadata, never paths); bytes live under a controlled root +for a short TTL. Size/MIME caps apply before any write; SHA-256 is re-verified on read; ``load`` needs +the exact scope key and consumes atomically. The index is lock-guarded and files are temp-written then +renamed so readers never see partials.""" from __future__ import annotations @@ -43,10 +25,8 @@ DEFAULT_ARTIFACT_TTL_SECONDS = 300.0 DEFAULT_MAX_ARTIFACT_BYTES = 10 * 1024 * 1024 #: Exact allowlist — parameterized/unknown variants are rejected. DEFAULT_ALLOWED_MIME_TYPES = frozenset({ - "application/json", "application/pdf", "image/gif", "image/jpeg", - "image/png", "image/webp", "text/plain", + "application/json", "application/pdf", "image/gif", "image/jpeg", "image/png", "image/webp", "text/plain", }) -_ARTIFACT_ID_HEX = 32 _ARTIFACT_ID_RE = re.compile(r"^[0-9a-f]{32}$") _TEMP_SUFFIX = ".tmp" @@ -86,7 +66,6 @@ class ArtifactTraversal(ArtifactError): @dataclass(frozen=True) class ArtifactReceipt: """Provenance record returned to the caller of ``store``.""" - artifact_id: str sha256: str size_bytes: int @@ -99,35 +78,22 @@ class ArtifactReceipt: def to_dict(self, *, download_path: str = "") -> dict[str, Any]: """Serialize to the wire receipt (never contains file paths).""" - receipt = { - "artifact_id": self.artifact_id, - "sha256": self.sha256, - "size_bytes": self.size_bytes, - "content_type": self.content_type, - "filename": self.filename, - "created_at": self.created_at, - "expires_at": self.expires_at, - "ttl_seconds": self.ttl_seconds, - "one_shot": True, + return { + "artifact_id": self.artifact_id, "sha256": self.sha256, "size_bytes": self.size_bytes, + "content_type": self.content_type, "filename": self.filename, "created_at": self.created_at, + "expires_at": self.expires_at, "ttl_seconds": self.ttl_seconds, "one_shot": True, + **({"download_path": download_path} if download_path else {}), } - if download_path: - receipt["download_path"] = download_path - return receipt def artifact_scope_key(scope: Any) -> str: """Derive the stable scope key an artifact is bound to. - Only principal (mandatory) + transport family participate. ``session_id`` - is deliberately EXCLUDED: HTTP artifact routes authenticate by API key and - can't resolve a session, while broker dispatch always carries one — hashing - it would make upload and dispatch never compose. Ids are unguessable and - downloads one-shot, so cross-session reuse within one principal is by - design. Capabilities/optional ids are excluded so a reconnect keeps its - artifacts. - """ - principal = "" - family = "" + Only principal (mandatory) + transport family participate. ``session_id`` is deliberately EXCLUDED: HTTP + artifact routes authenticate by API key and can't resolve a session while broker dispatch always carries + one, so hashing it would make upload and dispatch never compose (ids are unguessable and downloads + one-shot). Capabilities/optional ids are excluded so a reconnect keeps its artifacts.""" + principal = family = "" try: principal = str(getattr(scope, "principal_id", "") or "") family = str(getattr(scope, "transport_family", "") or "") @@ -136,12 +102,7 @@ def artifact_scope_key(scope: Any) -> str: if not principal: # Fail closed: only an authenticated principal may mint artifacts. raise ArtifactError("artifact scope must carry a resolved principal") - material = f"{principal}\x00{family}".encode("utf-8") - return hashlib.sha256(material).hexdigest() - - -def _sha256(data: bytes) -> str: - return hashlib.sha256(data).hexdigest() + return hashlib.sha256(f"{principal}\x00{family}".encode("utf-8")).hexdigest() @dataclass @@ -152,14 +113,8 @@ class _ArtifactEntry: class ArtifactStore: """Thread-safe, TTL-bounded, scope-bound one-shot artifact store.""" - - def __init__( - self, root: Path, *, - ttl_seconds: float = DEFAULT_ARTIFACT_TTL_SECONDS, - max_bytes: int = DEFAULT_MAX_ARTIFACT_BYTES, - allowed_mime_types: frozenset = DEFAULT_ALLOWED_MIME_TYPES, - clock: Optional[Callable[[], float]] = None, - ) -> None: + def __init__(self, root: Path, *, ttl_seconds: float = DEFAULT_ARTIFACT_TTL_SECONDS, max_bytes: int = DEFAULT_MAX_ARTIFACT_BYTES, + allowed_mime_types: frozenset = DEFAULT_ALLOWED_MIME_TYPES, clock: Optional[Callable[[], float]] = None) -> None: self._root = Path(root) self._root.mkdir(parents=True, exist_ok=True) self._ttl_seconds = max(1.0, float(ttl_seconds)) @@ -173,11 +128,7 @@ class ArtifactStore: self._sweep_orphan_files() def _sweep_orphan_files(self) -> None: - """Delete on-disk files with no live index entry. - - Only names matching the minted 32-hex id shape or the ``*.tmp`` - staging suffix are touched; anything else in the directory is left. - """ + """Delete on-disk files with no live index entry; only minted-id-shaped and ``*.tmp`` names are touched.""" try: candidates = list(self._root.iterdir()) except OSError: @@ -185,18 +136,10 @@ class ArtifactStore: with self._lock: live = set(self._entries) for path in candidates: - name = path.name - is_temp = name.endswith(_TEMP_SUFFIX) - if not path.is_file() or not (is_temp or _ARTIFACT_ID_RE.fullmatch(name)): - continue - if not is_temp and name in live: - continue - with contextlib.suppress(OSError): - path.unlink(missing_ok=True) - - # ------------------------------------------------------------------ - # Public API - # ------------------------------------------------------------------ + orphan = path.name.endswith(_TEMP_SUFFIX) or (_ARTIFACT_ID_RE.fullmatch(path.name) and path.name not in live) + if path.is_file() and orphan: + with contextlib.suppress(OSError): + path.unlink(missing_ok=True) @property def root(self) -> Path: @@ -212,11 +155,7 @@ class ArtifactStore: return self._allowed_mime_types def store(self, data: bytes, *, filename: str, content_type: str, scope: Any) -> ArtifactReceipt: - """Validate and store one artifact, returning its provenance receipt. - - Raises :class:`ArtifactTooLarge` / :class:`ArtifactMimeRejected` - before any disk write; :class:`ArtifactError` on an unresolved scope. - """ + """Validate and store one artifact, returning its receipt; size/MIME rejections fire before any disk write.""" size = len(data) if size > self._max_bytes: raise ArtifactTooLarge(f"artifact is {size} bytes; cap is {self._max_bytes}") @@ -225,23 +164,19 @@ class ArtifactStore: raise ArtifactMimeRejected(f"content type {content_type!r} is outside the exact allowlist") scope_key = artifact_scope_key(scope) now = self._clock() - # Mint a fresh id; retry on an astronomically unlikely collision. while True: - artifact_id = secrets.token_hex(_ARTIFACT_ID_HEX // 2) + artifact_id = secrets.token_hex(16) target = self._artifact_path(artifact_id) with self._lock: - if artifact_id in self._entries or target.exists(): - continue - receipt = ArtifactReceipt( - artifact_id=artifact_id, sha256=_sha256(data), size_bytes=size, - content_type=normalized_type, filename=_bounded_filename(filename), - created_at=now, expires_at=now + self._ttl_seconds, - ttl_seconds=self._ttl_seconds, scope_key=scope_key, - ) - self._entries[artifact_id] = _ArtifactEntry(receipt=receipt, path=target) - break - + if artifact_id not in self._entries and not target.exists(): + receipt = ArtifactReceipt( + artifact_id=artifact_id, sha256=hashlib.sha256(data).hexdigest(), size_bytes=size, + content_type=normalized_type, filename=_bounded_filename(filename), created_at=now, + expires_at=now + self._ttl_seconds, ttl_seconds=self._ttl_seconds, scope_key=scope_key, + ) + self._entries[artifact_id] = _ArtifactEntry(receipt=receipt, path=target) + break # Temp + atomic rename so readers never observe a partial artifact. temp = target.with_name(f"{target.name}{_TEMP_SUFFIX}") try: @@ -259,48 +194,39 @@ class ArtifactStore: return receipt def validate(self, artifact_id: str, *, scope: Any) -> ArtifactReceipt: - """Return the receipt when the artifact is live for ``scope`` - (existence, TTL, scope) without consuming it; raises otherwise.""" + """Receipt when the artifact is live for ``scope`` (existence, TTL, scope), without consuming.""" return self._entry_for(artifact_id, scope=scope).receipt def load(self, artifact_id: str, *, scope: Any) -> tuple[bytes, ArtifactReceipt]: - """One-shot download: verify, read, checksum, then consume. - - A second ``load`` raises :class:`ArtifactNotFound`. A checksum - mismatch raises :class:`ArtifactChecksumMismatch` without consuming. - """ + """One-shot download: verify, read, checksum, then consume (a checksum mismatch does not consume).""" with self._lock: entry = self._entry_for(artifact_id, scope=scope) - path = entry.path - if not path.exists(): + if not entry.path.exists(): self._entries.pop(artifact_id, None) raise ArtifactNotFound(f"artifact {artifact_id!r} is gone") try: - data = path.read_bytes() + data = entry.path.read_bytes() except OSError as exc: raise ArtifactError(f"artifact read failed: {exc}") from exc - if _sha256(data) != entry.receipt.sha256: + if hashlib.sha256(data).hexdigest() != entry.receipt.sha256: raise ArtifactChecksumMismatch(f"artifact {artifact_id!r} failed SHA-256 validation") # Drop the index entry first so a concurrent load fails closed. self._entries.pop(artifact_id, None) try: - path.unlink(missing_ok=True) + entry.path.unlink(missing_ok=True) except OSError: logger.warning("artifact %s: file removal failed; TTL sweep will retry", artifact_id) return data, entry.receipt def prune_expired(self, now: Optional[float] = None) -> int: - """Delete every artifact past its TTL (and stale temp files); return - the count removed. Idempotent.""" + """Delete every artifact past its TTL (and stale temp files); return the count removed.""" now = self._clock() if now is None else float(now) with self._lock: removed = self._prune_expired_locked(now) for temp in self._root.glob(f"*{_TEMP_SUFFIX}"): - try: + with contextlib.suppress(OSError): if temp.stat().st_mtime <= now - self._ttl_seconds: temp.unlink(missing_ok=True) - except OSError: - continue return removed def count(self) -> int: @@ -308,9 +234,11 @@ class ArtifactStore: with self._lock: return len(self._entries) - # ------------------------------------------------------------------ - # Internals - # ------------------------------------------------------------------ + def _discard_locked(self, artifact_id: str, path: Path) -> None: + """Drop the index entry and best-effort unlink its file.""" + self._entries.pop(artifact_id, None) + with contextlib.suppress(OSError): + path.unlink(missing_ok=True) def _entry_for(self, artifact_id: str, *, scope: Any) -> _ArtifactEntry: path = self._artifact_path(artifact_id) @@ -326,23 +254,17 @@ class ArtifactStore: if entry is None: raise ArtifactNotFound(f"unknown artifact {artifact_id!r}") if entry.receipt.expires_at <= now: - self._entries.pop(artifact_id, None) - with contextlib.suppress(OSError): - path.unlink(missing_ok=True) + self._discard_locked(artifact_id, path) raise ArtifactExpired(f"artifact {artifact_id!r} expired") if entry.receipt.scope_key != scope_key: raise ArtifactScopeMismatch(f"artifact {artifact_id!r} is bound to a different scope") return entry def _prune_expired_locked(self, now: float) -> int: - removed = 0 - for artifact_id, entry in list(self._entries.items()): - if entry.receipt.expires_at <= now: - self._entries.pop(artifact_id, None) - with contextlib.suppress(OSError): - entry.path.unlink(missing_ok=True) - removed += 1 - return removed + expired = [(aid, e.path) for aid, e in self._entries.items() if e.receipt.expires_at <= now] + for artifact_id, path in expired: + self._discard_locked(artifact_id, path) + return len(expired) def _artifact_path(self, artifact_id: str) -> Path: """Resolve a minted id strictly inside the controlled root.""" @@ -360,32 +282,18 @@ class ArtifactStore: def _normalize_content_type(value: str) -> str: """Return the canonical MIME type, or ``""`` for malformed input.""" - if not isinstance(value, str): - return "" - return value.strip().split(";", 1)[0].strip().lower() + return value.strip().split(";", 1)[0].strip().lower() if isinstance(value, str) else "" def _bounded_filename(value: str, limit: int = 160) -> str: """Sanitize a display-only filename; never used as a filesystem path.""" - if not isinstance(value, str): - return "" - cleaned = value.strip().replace("\\", "_").replace("/", "_") - cleaned = "".join(character for character in cleaned if ord(character) >= 32) - return cleaned[:limit] - - -# ---------------------------------------------------------------------- -# Rate limiting (route-level, per principal) -# ---------------------------------------------------------------------- + cleaned = value.strip().replace("\\", "_").replace("/", "_") if isinstance(value, str) else "" + return "".join(character for character in cleaned if ord(character) >= 32)[:limit] class ArtifactRateLimiter: """Sliding-window per-key limiter; the API server keys it by principal.""" - - def __init__( - self, *, window_seconds: float = 60.0, max_requests: int = 30, - clock: Optional[Callable[[], float]] = None, - ) -> None: + def __init__(self, *, window_seconds: float = 60.0, max_requests: int = 30, clock: Optional[Callable[[], float]] = None) -> None: self._window_seconds = max(1.0, float(window_seconds)) self._max_requests = max(1, int(max_requests)) self._clock = clock if clock is not None else time.time @@ -397,9 +305,8 @@ class ArtifactRateLimiter: if not isinstance(key, str) or not key: return False now = self._clock() - window_start = now - self._window_seconds with self._lock: - hits = [hit for hit in self._hits.get(key, []) if hit > window_start] + hits = [hit for hit in self._hits.get(key, []) if hit > now - self._window_seconds] allowed = len(hits) < self._max_requests if allowed: hits.append(now) diff --git a/gateway/browser_control_broker.py b/gateway/browser_control_broker.py index 5319873a75..58e2d7c672 100644 --- a/gateway/browser_control_broker.py +++ b/gateway/browser_control_broker.py @@ -1,34 +1,10 @@ -"""Transport-neutral browser-control broker core. - -Binds an identity-scoped *controller* (the party driving a browser) to -*callers* (agents talking to it over any transport) without knowing HTTP, -WebSocket, or any wire format. The browser is a stateful single-owner -resource while the agent side is multi-tenant and multi-transport, so every -rule here makes a class of violation structurally impossible: - -- Registration tickets are short-lived, single-use, identity-bound, and - ``secrets``-random; ``consume_ticket`` exchanges one exactly once. The - ticket is the only cross-transport credential minted here; transports - decide how to carry it. -- ``select`` matches on *every* stable identity field (principal, profile, - session, controller id, browser profile id, transport family) plus the - controller's currently negotiated capability set; partial matches yield - ``None``. A same-identity reconnect renegotiates capabilities in place. -- One pending command per command id; ``complete`` is single-shot (late - completion after cancel/detach returns ``False``). -- ``cancel`` aborts only the command matching scope + tool_call_id. -- ``detach`` fails closed: pending work of that scope raises - :class:`ControllerCancelled` instead of racing a late ``complete``. -- ``disconnect`` marks a transport offline without cancelling running work; - re-attaching the same identity refreshes the callback and may still - deliver the original result. Detach, cancel, identity replacement, and - timeout remain terminal. - -Thread-safety: all state transitions happen under one reentrant lock; the -send callback is invoked *outside* it so a controller may synchronously -``complete`` from inside its own send, and waiters park on per-command -events, never on the broker lock. -""" +"""Transport-neutral browser-control broker core: binds an identity-scoped *controller* +(the party driving a browser) to *callers* on any transport. Tickets are short-lived, +single-use, identity-bound and consumed exactly once; ``select`` matches every stable +identity field plus the current capability set; ``complete`` is single-shot; ``detach`` +fails pending work closed while ``disconnect`` only marks the transport offline. State +changes happen under one RLock; the send callback runs *outside* it so a controller may +``complete`` from inside its own send.""" from __future__ import annotations @@ -48,39 +24,18 @@ DEFAULT_TICKET_TTL = 30.0 DEFAULT_COMMAND_TIMEOUT = 30.0 #: Maximum cancel frames retained while a same-identity controller is offline. MAX_DEFERRED_CANCELS = 512 - -#: Current wire protocol version. Registration requires this exact integer; -#: booleans are rejected even though ``bool`` subclasses ``int``. +#: Current wire protocol version; registration requires this exact int (bools rejected). BROWSER_CONTROL_PROTOCOL_VERSION = 1 -#: Exact controller capability allowlist shared by every transport. Raw CDP, -#: script evaluation, console access, and other privileged surfaces stay out. -BROWSER_CONTROL_CAPABILITIES = frozenset( - { - "controller.noop", - "browser_back", - "browser_click", - "browser_navigate", - "browser_press", - "browser_screenshot", - "browser_scroll", - "browser_snapshot", - "browser_tab_activate", - "browser_tabs", - "browser_type", - } -) - -#: Privileged capabilities (JS eval, raw CDP): fail-closed unless Developer -#: Mode (``browser.extension_control.developer_mode``) is on AND the -#: controller explicitly negotiated them. +#: Exact controller capability allowlist shared by every transport (raw CDP/eval/console stay out). +BROWSER_CONTROL_CAPABILITIES = frozenset({ + "controller.noop", "browser_back", "browser_click", "browser_navigate", "browser_press", "browser_screenshot", + "browser_scroll", "browser_snapshot", "browser_tab_activate", "browser_tabs", "browser_type", +}) +#: Privileged capabilities: fail-closed unless Developer Mode is on AND explicitly negotiated. BROWSER_CONTROL_DEVELOPER_CAPABILITIES = frozenset({"browser_cdp", "browser_evaluate"}) - -#: Artifact transport capabilities. Non-developer because payloads never ride -#: in controller frames: only a store-validated ``artifact_id`` is dispatched. -BROWSER_CONTROL_ARTIFACT_CAPABILITIES = frozenset( - {"browser_artifact_download", "browser_artifact_upload"} -) +#: Artifact transport capabilities; non-developer because only a store-validated ``artifact_id`` travels. +BROWSER_CONTROL_ARTIFACT_CAPABILITIES = frozenset({"browser_artifact_download", "browser_artifact_upload"}) #: Wire method names for controller frames; transports carry them verbatim. FRAME_COMMAND = "browser.controller.command" @@ -96,22 +51,14 @@ def _extension_control_flag(config: Optional[dict], key: str) -> bool: """Read ``browser.extension_control.`` as a literal ``True`` (default off).""" if config is None: try: - # Hot path (every browser tool call / check_fn): the read-only - # loader skips load_config()'s deepcopy; we never mutate. + # Hot path (every browser tool call): the read-only loader skips load_config()'s deepcopy. from hermes_cli.config import load_config_readonly - config = load_config_readonly() except Exception: return False - if not isinstance(config, dict): - return False - browser = config.get("browser") - if not isinstance(browser, dict): - return False - extension_control = browser.get("extension_control") - if not isinstance(extension_control, dict): - return False - return extension_control.get(key, False) is True + browser = config.get("browser") if isinstance(config, dict) else None + extension_control = browser.get("extension_control") if isinstance(browser, dict) else None + return isinstance(extension_control, dict) and extension_control.get(key, False) is True def browser_control_developer_mode(config: Optional[dict] = None) -> bool: @@ -124,25 +71,13 @@ def browser_control_enabled(config: Optional[dict] = None) -> bool: return _extension_control_flag(config, "enabled") -def filter_browser_control_capabilities( - value: Any, - *, - developer_mode: Optional[bool] = None, -) -> frozenset: - """Return the permitted subset of a JSON/RPC capability list. - - Non-list input has no capabilities; unknown/non-string entries are - dropped (registration rejects an empty returned set). Developer - capabilities pass only when Developer Mode is on (passed in, else read - from live config). - """ +def filter_browser_control_capabilities(value: Any, *, developer_mode: Optional[bool] = None) -> frozenset: + """Permitted subset of a capability list (non-list -> empty); developer caps only in Developer Mode.""" if not isinstance(value, list): return frozenset() allowed = BROWSER_CONTROL_CAPABILITIES | BROWSER_CONTROL_ARTIFACT_CAPABILITIES - if developer_mode is None: - developer_mode = browser_control_developer_mode() - if developer_mode is True: - allowed = allowed | BROWSER_CONTROL_DEVELOPER_CAPABILITIES + if (browser_control_developer_mode() if developer_mode is None else developer_mode) is True: + allowed |= BROWSER_CONTROL_DEVELOPER_CAPABILITIES return frozenset(c for c in value if isinstance(c, str) and c in allowed) @@ -173,7 +108,6 @@ class ControllerRejected(BrowserControlError): @dataclass(frozen=True) class ControllerScope: """Exact controller identity plus capability set; equality is over all fields.""" - principal_id: Optional[str] = None profile_id: Optional[str] = None session_id: Optional[str] = None @@ -183,26 +117,17 @@ class ControllerScope: capabilities: frozenset = frozenset() -def _scope_identity(scope: ControllerScope) -> tuple: - """Stable identity fields, excluding negotiated capabilities.""" - return ( - scope.principal_id, - scope.profile_id, - scope.session_id, - scope.controller_id, - scope.browser_profile_id, - scope.transport_family, - ) +#: Stable identity fields; negotiated ``capabilities`` are deliberately excluded. +_IDENTITY_FIELDS = ("principal_id", "profile_id", "session_id", "controller_id", "browser_profile_id", "transport_family") def _same_scope_identity(first: ControllerScope, second: ControllerScope) -> bool: - return _scope_identity(first) == _scope_identity(second) + return all(getattr(first, name) == getattr(second, name) for name in _IDENTITY_FIELDS) @dataclass(frozen=True) class Ticket: """Opaque, single-use registration credential.""" - value: str expires_at: float @@ -221,9 +146,7 @@ class _Controller: owner: Any = None connected: bool = True deferred_cancels: list[dict] = field(default_factory=list) - # Serializes command/cancel writes with detach or replacement. Broker - # state is never held while waiting on it, so a transport callback may - # synchronously complete() without deadlocking. + # Serializes command/cancel writes with detach or replacement; never held with broker state. send_lock: threading.Lock = field(default_factory=threading.Lock) @@ -239,20 +162,14 @@ class _PendingCommand: result: Any = None +def _cancel_frame(pending: _PendingCommand) -> dict: + return {"method": FRAME_CANCEL, "params": {"command_id": pending.command_id, "tool_call_id": pending.tool_call_id}} + + class BrowserControlBroker: - """Thread-safe broker core binding controllers to callers. - - ``clock`` is injectable (default ``time.monotonic``) so tests pin expiry. - """ - - def __init__( - self, - *, - ticket_ttl: float = DEFAULT_TICKET_TTL, - command_timeout: float = DEFAULT_COMMAND_TIMEOUT, - clock: Optional[Callable[[], float]] = None, - developer_mode: Optional[bool] = None, - ) -> None: + """Thread-safe broker core; ``clock`` is injectable (default ``time.monotonic``).""" + def __init__(self, *, ticket_ttl: float = DEFAULT_TICKET_TTL, command_timeout: float = DEFAULT_COMMAND_TIMEOUT, + clock: Optional[Callable[[], float]] = None, developer_mode: Optional[bool] = None) -> None: self._ticket_ttl = ticket_ttl self._command_timeout = command_timeout self._clock = clock if clock is not None else time.monotonic @@ -260,19 +177,15 @@ class BrowserControlBroker: self._tickets: Dict[str, _TicketRecord] = {} self._controllers: Dict[ControllerScope, _Controller] = {} self._pending: Dict[str, _PendingCommand] = {} - # None defers to live config on every selection so flipping - # developer_mode off REVOKES raw CDP/eval from attached controllers - # without restart; an explicit bool pins the gate (tests, multi-tenant). - self._developer_mode_pinned: Optional[bool] = ( - None if developer_mode is None else developer_mode is True - ) + # None defers to live config on every selection (so flipping developer_mode off REVOKES + # raw CDP/eval from attached controllers without restart); a bool pins the gate. + self._developer_mode_pinned: Optional[bool] = None if developer_mode is None else developer_mode is True # Artifact stores keyed by profile id; ``None`` is the default slot. - # Per-profile stores keep multiplex profile A from pinning profile B - # to A's physical root. self._artifact_stores: Dict[Optional[str], Any] = {} - def _developer_mode_now(self) -> bool: - """Current Developer Mode authority (live config unless pinned).""" + @property + def developer_mode(self) -> bool: + """Whether privileged capabilities may be selected/dispatched (live config unless pinned).""" if self._developer_mode_pinned is not None: return self._developer_mode_pinned try: @@ -280,55 +193,29 @@ class BrowserControlBroker: except Exception: return False - @property - def developer_mode(self) -> bool: - """Whether privileged capabilities may be selected/dispatched.""" - return self._developer_mode_now() - - def attach_artifact_store( - self, store: Any, *, profile_id: Optional[str] = None - ) -> None: - """Attach a store exposing ``validate(artifact_id, *, scope) -> receipt`` - (raising the artifacts module's :class:`ArtifactError` subclasses). - - ``profile_id`` scopes the store to one profile on multiplex hosts; - ``None`` registers the default store. ``store=None`` clears the slot. - Artifact actions without a resolvable store fail closed. - """ + def attach_artifact_store(self, store: Any, *, profile_id: Optional[str] = None) -> None: + """Attach a store exposing ``validate(artifact_id, *, scope) -> receipt`` for one profile + (``None`` = default slot); ``store=None`` clears the slot. Artifact actions fail closed without one.""" if store is None: self._artifact_stores.pop(profile_id, None) - return - self._artifact_stores[profile_id] = store + else: + self._artifact_stores[profile_id] = store def _artifact_store_for_scope(self, scope: "ControllerScope") -> Any: - """Exact profile-scoped store, else the default (``None``) slot so - single-profile hosts keep the historical one-store behaviour.""" - profile = getattr(scope, "profile_id", None) or None - store = self._artifact_stores.get(profile) + store = self._artifact_stores.get(getattr(scope, "profile_id", None) or None) return store if store is not None else self._artifact_stores.get(None) - # ------------------------------------------------------------------ - # Registration tickets - # ------------------------------------------------------------------ - def mint_ticket(self, scope: ControllerScope) -> Ticket: """Mint a short-lived, single-use ticket bound to ``scope``.""" now = self._clock() with self._lock: - for stale in [v for v, rec in self._tickets.items() if rec.expires_at <= now]: - del self._tickets[stale] + self._tickets = {v: rec for v, rec in self._tickets.items() if rec.expires_at > now} value = secrets.token_urlsafe(32) - record = _TicketRecord(scope=scope, expires_at=now + self._ticket_ttl) - self._tickets[value] = record + self._tickets[value] = record = _TicketRecord(scope=scope, expires_at=now + self._ticket_ttl) return Ticket(value=value, expires_at=record.expires_at) def consume_ticket(self, value: str) -> ControllerScope: - """Exchange a ticket for its scope exactly once. - - Raises :class:`ControllerTicketInvalid` for unknown, already-consumed, - or expired tickets; expiry is checked against the live clock at - consume time. - """ + """Exchange a ticket for its scope exactly once; unknown/consumed/expired -> ControllerTicketInvalid.""" now = self._clock() with self._lock: record = self._tickets.get(value) @@ -341,52 +228,26 @@ class BrowserControlBroker: record.consumed = True return record.scope - # ------------------------------------------------------------------ - # Controller registration / selection - # ------------------------------------------------------------------ - def _controller_for_identity_locked(self, scope: ControllerScope) -> Optional[_Controller]: """Attached controller sharing ``scope``'s stable identity (any capabilities).""" - return next( - (c for c in self._controllers.values() if _same_scope_identity(c.scope, scope)), - None, - ) + return next((c for c in self._controllers.values() if _same_scope_identity(c.scope, scope)), None) - def _lane_scopes_locked(self, session_id: str, principal_id: str, family: str) -> list[ControllerScope]: - """Attached scopes bound to one session lane (session + principal + transport).""" - return [ - scope - for scope in self._controllers - if scope.session_id == session_id - and scope.principal_id == principal_id - and scope.transport_family == family - ] + def _live_controller(self, scope: ControllerScope) -> Optional[_Controller]: + with self._lock: + controller = self._controller_for_identity_locked(scope) + return controller if controller is not None and controller.connected else None - def attach( - self, - scope: ControllerScope, - send: Callable[[dict], None], - *, - owner: Any = None, - ) -> None: - """Attach or refresh the controller for one stable identity. - - A same-identity reconnect refreshes the send callback and negotiated - capabilities without cancelling pending work. A different controller - or browser profile in the same authenticated session lane - hard-replaces the previous identity. - """ + def attach(self, scope: ControllerScope, send: Callable[[dict], None], *, owner: Any = None) -> None: + """Attach or refresh the controller for one stable identity: a same-identity reconnect refreshes send + and capabilities without cancelling pending work; a different controller/browser profile in the same + authenticated session lane hard-replaces it.""" while True: with self._lock: existing = self._controller_for_identity_locked(scope) lane_scopes = [ - candidate - for candidate in self._controllers - if candidate.principal_id == scope.principal_id - and candidate.profile_id == scope.profile_id - and candidate.session_id == scope.session_id - and candidate.transport_family == scope.transport_family - and not _same_scope_identity(candidate, scope) + c for c in self._controllers if not _same_scope_identity(c, scope) + and (c.principal_id, c.profile_id, c.session_id, c.transport_family) + == (scope.principal_id, scope.profile_id, scope.session_id, scope.transport_family) ] if existing is None and not lane_scopes: self._controllers[scope] = _Controller(scope=scope, send=send, owner=owner) @@ -404,69 +265,41 @@ class BrowserControlBroker: if self._controllers.get(existing.scope) is not existing: continue self._controllers.pop(existing.scope, None) - existing.scope = scope - existing.send = send - existing.owner = owner - existing.connected = False - for pending in self._pending.values(): - if _same_scope_identity(pending.scope, scope): - pending.scope = scope - deferred = list(existing.deferred_cancels) - existing.deferred_cancels.clear() + existing.scope, existing.send, existing.owner, existing.connected = scope, send, owner, False + for pending in self._pending_for_scope_locked(scope): + pending.scope = scope + deferred, existing.deferred_cancels = existing.deferred_cancels, [] self._controllers[scope] = existing - unsent: list[dict] = [] for index, frame in enumerate(deferred): try: send(frame) + continue except Exception: logger.exception("failed to flush deferred browser-controller cancel") - unsent = deferred[index:] - break - if unsent: with self._lock: if self._controllers.get(scope) is existing: - existing.deferred_cancels = unsent[-MAX_DEFERRED_CANCELS:] - raise ConnectionError( - "browser controller reconnect could not flush deferred cancels" - ) + existing.deferred_cancels = deferred[index:][-MAX_DEFERRED_CANCELS:] + raise ConnectionError("browser controller reconnect could not flush deferred cancels") with self._lock: if self._controllers.get(scope) is existing: existing.connected = True return def select(self, scope: ControllerScope, capability: str) -> Optional[_Controller]: - """Return the connected controller matching identity and capability. - - The caller's capabilities are not authoritative: identity is matched, - then the controller's *current* negotiated set is checked. Offline - controllers never accept new dispatches. Developer capabilities are - additionally gated on the LIVE Developer Mode flag (unless pinned). - """ - if capability in BROWSER_CONTROL_DEVELOPER_CAPABILITIES and not self._developer_mode_now(): + """Connected controller matching identity whose *current* negotiated set holds ``capability`` (the + caller's set is not authoritative); developer capabilities are also gated on LIVE Developer Mode.""" + if capability in BROWSER_CONTROL_DEVELOPER_CAPABILITIES and not self.developer_mode: return None - with self._lock: - controller = self._controller_for_identity_locked(scope) - if controller is None or not controller.connected: - return None - return controller if capability in controller.scope.capabilities else None + controller = self._live_controller(scope) + return controller if controller is not None and capability in controller.scope.capabilities else None def is_owner(self, scope: ControllerScope, owner: Any) -> bool: - """Whether ``owner`` is the exact live transport for ``scope``. + """Whether ``owner`` is the exact live transport for ``scope`` (capability-independent).""" + controller = self._live_controller(scope) + return controller is not None and controller.owner is owner - Independent of capabilities, so a least-privilege controller need not - request ``controller.noop`` merely to heartbeat or complete an action. - """ - with self._lock: - controller = self._controller_for_identity_locked(scope) - return controller is not None and controller.connected and controller.owner is owner - - def disconnect( - self, - scope: ControllerScope, - *, - owner: Any = _OWNER_UNSET, - ) -> bool: + def disconnect(self, scope: ControllerScope, *, owner: Any = _OWNER_UNSET) -> bool: """Mark one exact controller transport offline without cancelling work.""" with self._lock: controller = self._controller_for_identity_locked(scope) @@ -474,37 +307,21 @@ class BrowserControlBroker: return False with controller.send_lock: with self._lock: - if self._controllers.get(controller.scope) is not controller: + owned = owner is _OWNER_UNSET or controller.owner is owner + if self._controllers.get(controller.scope) is not controller or not owned: return False - if owner is not _OWNER_UNSET and controller.owner is not owner: - return False - controller.connected = False - controller.owner = None + controller.connected, controller.owner = False, None return True - def detach( - self, - scope: ControllerScope, - *, - owner: Any = _OWNER_UNSET, - notify_controller: bool = True, - ) -> None: - """Remove the controller for ``scope`` and fail its pending work closed. - - Pending commands resolve cancelled (dispatchers raise - :class:`ControllerCancelled`); a late ``complete`` returns ``False``. - """ + def detach(self, scope: ControllerScope, *, owner: Any = _OWNER_UNSET, notify_controller: bool = True) -> None: + """Remove the controller for ``scope`` and fail its pending work closed (ControllerCancelled).""" with self._lock: controller = self._controllers.get(scope) - if controller is None: - return - if owner is not _OWNER_UNSET and controller.owner != owner: + if controller is None or (owner is not _OWNER_UNSET and controller.owner != owner): return with controller.send_lock: with self._lock: - if self._controllers.get(scope) is not controller: - return - if owner is not _OWNER_UNSET and controller.owner != owner: + if self._controllers.get(scope) is not controller or (owner is not _OWNER_UNSET and controller.owner != owner): return self._controllers.pop(scope, None) pendings = self._pending_for_scope_locked(scope) @@ -515,65 +332,31 @@ class BrowserControlBroker: if notify_controller: self._emit_cancel_frames(controller, pendings) - # ------------------------------------------------------------------ - # Command lifecycle - # ------------------------------------------------------------------ - def dispatch( - self, - scope: ControllerScope, - *, - action: str, - arguments: Optional[dict] = None, - tool_call_id: Optional[str] = None, + self, scope: ControllerScope, *, action: str, arguments: Optional[dict] = None, tool_call_id: Optional[str] = None, ) -> Any: - """Send one controller command and block for its completion. - - Raises :class:`ControllerUnavailable` (no exact match), - :class:`ControllerCancelled`, :class:`ControllerTimeout`, or - :class:`ControllerRejected` (``ok=False``). Artifact actions also - require an attached store and an approved ``artifact_id`` argument; - only the id travels in the frame, never the payload. - """ + """Send one controller command and block for completion; raises ControllerUnavailable/Cancelled/Timeout/ + Rejected. Artifact actions also need an attached store and an approved ``artifact_id`` (only the id travels).""" controller = self.select(scope, action) if controller is None: - raise ControllerUnavailable( - f"no controller for scope {scope!r} with capability {action!r}" - ) - + raise ControllerUnavailable(f"no controller for scope {scope!r} with capability {action!r}") arguments = dict(arguments or {}) if action in BROWSER_CONTROL_ARTIFACT_CAPABILITIES: self._validate_artifact_reference(scope, action, arguments) - command_id = secrets.token_hex(16) - frame = { - "method": FRAME_COMMAND, - "params": { - "command_id": command_id, - "action": action, - "arguments": arguments, - "controller_id": scope.controller_id, - "browser_profile_id": scope.browser_profile_id, - "tool_call_id": tool_call_id, - }, - } - pending = _PendingCommand( - scope=controller.scope, - command_id=command_id, - tool_call_id=tool_call_id, - ) + frame = {"method": FRAME_COMMAND, "params": { + "command_id": command_id, "action": action, "arguments": arguments, "controller_id": scope.controller_id, + "browser_profile_id": scope.browser_profile_id, "tool_call_id": tool_call_id, + }} + pending = _PendingCommand(scope=controller.scope, command_id=command_id, tool_call_id=tool_call_id) with controller.send_lock: with self._lock: # select() ran outside the send lock; revalidate the live # controller so disconnect/replacement can't strand a command. - attached = self._controller_for_identity_locked(scope) - if attached is not controller or not controller.connected: - raise ControllerUnavailable( - f"controller for scope {scope!r} detached before dispatch" - ) + if self._controller_for_identity_locked(scope) is not controller or not controller.connected: + raise ControllerUnavailable(f"controller for scope {scope!r} detached before dispatch") pending.scope = controller.scope self._pending[command_id] = pending - try: controller.send(frame) except Exception: @@ -583,14 +366,13 @@ class BrowserControlBroker: raise if not pending.event.wait(timeout=self._command_timeout): - timed_out = False with self._lock: # Event.wait() may return False at the exact boundary where a # completion already won and removed the pending command. - if not pending.done and self._pending.get(command_id) is pending: + timed_out = not pending.done and self._pending.get(command_id) is pending + if timed_out: pending.done = True del self._pending[command_id] - timed_out = True if timed_out: with controller.send_lock: with self._lock: @@ -600,97 +382,48 @@ class BrowserControlBroker: active = None if active is not None: self._emit_cancel_frames(active, [pending]) - raise ControllerTimeout( - f"controller did not complete command {command_id!r} " - f"within {self._command_timeout}s" - ) - + raise ControllerTimeout(f"controller did not complete command {command_id!r} within {self._command_timeout}s") if pending.cancelled: raise ControllerCancelled(f"command {command_id!r} was cancelled") if not pending.ok: - raise ControllerRejected( - f"controller rejected command {command_id!r}: {pending.result!r}" - ) + raise ControllerRejected(f"controller rejected command {command_id!r}: {pending.result!r}") return pending.result - def complete( - self, - command_id: str, - *, - scope: Optional[ControllerScope] = None, - ok: bool, - result: Any = None, - ) -> bool: - """Resolve a pending command by id; ``False`` when none is pending. - - Safe from inside the controller's own ``send`` callback. Late - completions after ``cancel``/``detach`` are ignored (``False``). - """ + def complete(self, command_id: str, *, scope: Optional[ControllerScope] = None, ok: bool, result: Any = None) -> bool: + """Resolve a pending command by id; ``False`` when none is pending. Safe from inside the send callback.""" with self._lock: pending = self._pending.get(command_id) - if pending is None or pending.done: + if pending is None or pending.done or (scope is not None and pending.scope != scope): return False - if scope is not None and pending.scope != scope: - return False - pending.done = True - pending.ok = ok is True - pending.result = result + pending.done, pending.ok, pending.result = True, ok is True, result del self._pending[command_id] pending.event.set() return True def cancel(self, scope: ControllerScope, *, tool_call_id: Optional[str]) -> bool: - """Cancel exactly the pending command matching ``scope`` + tool_call_id. - - Emits one cancel frame; returns ``False`` when nothing matched so - transports can answer idempotently. - """ - with self._lock: - controller = self._controller_for_identity_locked(scope) - if controller is None or not controller.connected: + """Cancel the pending command matching ``scope`` + tool_call_id (one cancel frame); ``False`` if none.""" + controller = self._live_controller(scope) + if controller is None: return False with controller.send_lock: with self._lock: - attached = self._controller_for_identity_locked(scope) - if attached is not controller or not controller.connected: + if self._controller_for_identity_locked(scope) is not controller or not controller.connected: return False - target = next( - ( - p - for p in self._pending.values() - if _same_scope_identity(p.scope, scope) - and p.tool_call_id == tool_call_id - and not p.done - ), - None, - ) + target = next((p for p in self._pending_for_scope_locked(scope) + if p.tool_call_id == tool_call_id and not p.done), None) if target is None: return False self._resolve_pending(target, cancelled=True) self._emit_cancel_frames(controller, [target]) return True - # ------------------------------------------------------------------ - # Internals (callers hold the lock unless noted) - # ------------------------------------------------------------------ - def _resolve_pending(self, pending: _PendingCommand, *, cancelled: bool) -> None: - pending.cancelled = cancelled - pending.done = True + pending.cancelled, pending.done = cancelled, True del self._pending[pending.command_id] pending.event.set() - def _validate_artifact_reference( - self, - scope: ControllerScope, - action: str, - arguments: dict, - ) -> None: - """Fail closed unless ``arguments`` carries a store-approved artifact id. - - Missing store/id, traversal, expiry, checksum, or scope mismatch all - surface as :class:`ControllerRejected` before any frame is emitted. - """ + def _validate_artifact_reference(self, scope: ControllerScope, action: str, arguments: dict) -> None: + """Fail closed unless ``arguments`` carries a store-approved artifact id (failures -> ControllerRejected).""" store = self._artifact_store_for_scope(scope) if store is None: raise ControllerRejected(f"{action} requires an attached artifact store") @@ -702,22 +435,10 @@ class BrowserControlBroker: except ControllerRejected: raise except Exception as exc: - raise ControllerRejected( - f"{action} rejected artifact reference {artifact_id!r}: {exc}" - ) from exc - - @staticmethod - def _cancel_frame(pending: _PendingCommand) -> dict: - return { - "method": FRAME_CANCEL, - "params": { - "command_id": pending.command_id, - "tool_call_id": pending.tool_call_id, - }, - } + raise ControllerRejected(f"{action} rejected artifact reference {artifact_id!r}: {exc}") from exc def _defer_cancel_locked(self, controller: _Controller, pending: _PendingCommand) -> None: - controller.deferred_cancels.append(self._cancel_frame(pending)) + controller.deferred_cancels.append(_cancel_frame(pending)) if len(controller.deferred_cancels) > MAX_DEFERRED_CANCELS: del controller.deferred_cancels[:-MAX_DEFERRED_CANCELS] @@ -728,65 +449,30 @@ class BrowserControlBroker: """Send cancel frames (caller holds ``send_lock``, never the broker lock).""" for pending in pendings: try: - controller.send(self._cancel_frame(pending)) + controller.send(_cancel_frame(pending)) except Exception: - logger.exception( - "failed to emit cancel frame for command %r", pending.command_id - ) + logger.exception("failed to emit cancel frame for command %r", pending.command_id) - # ------------------------------------------------------------------ - # Session-lane lookups / bulk teardown - # ------------------------------------------------------------------ - - @staticmethod - def _lane_key(session_id, task_id, principal_id, transport_family) -> Optional[tuple[str, str, str]]: - target = str(session_id or task_id or "").strip() - principal = str(principal_id or "").strip() - family = str(transport_family or "").strip() - if not target or not principal or not family: - return None - return target, principal, family - - def scope_for_session( - self, - *, - session_id: Optional[str] = None, - task_id: Optional[str] = None, - principal_id: Optional[str] = None, - transport_family: Optional[str] = None, - ) -> Optional[ControllerScope]: - """Return one unambiguous attached scope for a server-owned session. - - The public session id is only a hint; the caller must also supply its - server-derived principal and transport family. Missing identity, no - match, or multiple matches fail closed rather than picking by order. - """ - key = self._lane_key(session_id, task_id, principal_id, transport_family) - if key is None: - return None + def _lane_scopes(self, session_id, task_id, principal_id, transport_family) -> list[ControllerScope]: + """Attached scopes bound to one session lane (session + principal + transport).""" + key = tuple(str(v or "").strip() for v in (session_id or task_id, principal_id, transport_family)) + if not all(key): + return [] with self._lock: - matches = self._lane_scopes_locked(*key) + return [s for s in self._controllers if (s.session_id, s.principal_id, s.transport_family) == key] + + def scope_for_session(self, *, session_id: Optional[str] = None, task_id: Optional[str] = None, + principal_id: Optional[str] = None, transport_family: Optional[str] = None) -> Optional[ControllerScope]: + """One unambiguous attached scope for a server-owned session (session id is only a hint; the caller + supplies its server-derived principal + transport family). Missing/ambiguous identity fails closed.""" + matches = self._lane_scopes(session_id, task_id, principal_id, transport_family) return matches[0] if len(matches) == 1 else None - def lane_registered( - self, - *, - session_id: Optional[str] = None, - task_id: Optional[str] = None, - principal_id: Optional[str] = None, - transport_family: Optional[str] = None, - ) -> bool: - """Whether ANY controller (even offline) registered for this lane. - - Distinguishes "lane bound but currently unavailable" (fail closed; - the extension lane stays authoritative) from "never registered" - (caller keeps the legacy backend). Ambiguous lanes report True. - """ - key = self._lane_key(session_id, task_id, principal_id, transport_family) - if key is None: - return False - with self._lock: - return bool(self._lane_scopes_locked(*key)) + def lane_registered(self, *, session_id: Optional[str] = None, task_id: Optional[str] = None, + principal_id: Optional[str] = None, transport_family: Optional[str] = None) -> bool: + """Whether ANY controller (even offline) registered for this lane: "bound but unavailable" fails closed + vs "never registered" (caller keeps the legacy backend). Ambiguous lanes report True.""" + return bool(self._lane_scopes(session_id, task_id, principal_id, transport_family)) def disconnect_owner(self, owner: Any) -> int: """Mark every controller owned by one lost transport offline.""" diff --git a/gateway/cgroup_cleanup.py b/gateway/cgroup_cleanup.py index a68a971c37..8e34fc5a89 100644 --- a/gateway/cgroup_cleanup.py +++ b/gateway/cgroup_cleanup.py @@ -3,15 +3,14 @@ Runs as ``ExecStopPost=`` after the gateway's main process has exited: the safety net for long-lived helpers the gateway doesn't track (``adb``, platform bridges) that would otherwise be orphaned in the cgroup and block -``Restart=always``. - -Per-PID SIGKILLs over ``cgroup.procs`` are used deliberately instead of writing -``1`` to ``cgroup.kill``: the kernel has returned ``EINVAL`` on the cgroup-wide -kill while per-PID signal delivery still works. +``Restart=always``. Per-PID SIGKILLs over ``cgroup.procs`` are used instead of +writing ``1`` to ``cgroup.kill``: the kernel has returned ``EINVAL`` on the +cgroup-wide kill while per-PID signal delivery still works. """ from __future__ import annotations +import contextlib import os import re import signal @@ -36,23 +35,19 @@ def _read_cgroup_pids(cgroup_path: str) -> list[int]: return [] pids: list[int] = [] for line in raw.splitlines(): - try: + with contextlib.suppress(ValueError): pids.append(int(line.strip())) - except ValueError: - continue return pids def reap_cgroup(cgroup_path: str | None = None) -> int: """SIGKILL every PID in the cgroup other than the caller. Returns the count killed.""" - if cgroup_path is None: - cgroup_path = _own_cgroup_path() + cgroup_path = _own_cgroup_path() if cgroup_path is None else cgroup_path if not cgroup_path: return 0 - own = os.getpid() killed = 0 for pid in _read_cgroup_pids(cgroup_path): - if pid == own: + if pid == os.getpid(): continue try: os.kill(pid, signal.SIGKILL) # windows-footgun: ok — Linux-only (reads /proc, /sys/fs/cgroup; runs from a systemd unit) diff --git a/gateway/channel_directory.py b/gateway/channel_directory.py index b7558325e1..edac38888e 100644 --- a/gateway/channel_directory.py +++ b/gateway/channel_directory.py @@ -5,6 +5,7 @@ send_message reads it for action="list" and to resolve friendly channel names to """ import asyncio +import contextlib import json import logging import time @@ -75,12 +76,10 @@ def _apply_channel_aliases(platforms: Dict[str, Any]) -> None: continue chat_id = str(chat_id) friendly = friendly.strip() - matched = False - for e in entries: - if isinstance(e, dict) and e.get("id") == chat_id: - e["name"] = friendly - matched = True - if not matched: + matches = [e for e in entries if isinstance(e, dict) and e.get("id") == chat_id] + for e in matches: + e["name"] = friendly + if not matches: entries.append({ "id": chat_id, "name": friendly, "type": "group" if chat_id.endswith("@g.us") else "dm", "thread_id": None, @@ -92,7 +91,7 @@ def _normalize_channel_query(value: str) -> str: def _channel_target_name(platform_name: str, channel: Dict[str, Any]) -> str: - """Return the human-facing target label shown to users for a channel entry.""" + """Human-facing target label for a channel entry.""" name = channel["name"] if platform_name == "discord" and channel.get("guild"): return f"#{name}" @@ -118,8 +117,11 @@ def _session_entry_name(origin: Dict[str, Any]) -> str: return f"{base_name} / {topic_label}" -def _warn_slack_directory(team_id: str, detail: str) -> None: - """Warn once per team/error per interval for recurring Slack refresh failures.""" +def _report_slack_failure(team_id: str, error_code: Optional[str], detail: str) -> None: + """missing_scope is expected (session-history fallback); anything else warns once per interval.""" + if error_code == "missing_scope": + logger.debug("Channel directory: Slack team %s lacks channels:read; using session history only", team_id) + return key = (str(team_id), str(detail)) now = time.monotonic() last = _slack_directory_warning_last.get(key) @@ -130,14 +132,6 @@ def _warn_slack_directory(team_id: str, detail: str) -> None: logger.debug("Channel directory: suppressed repeated Slack channel list failure for team %s: %s", team_id, detail) -def _report_slack_failure(team_id: str, error_code: Optional[str], detail: str) -> None: - """missing_scope is expected (session-history fallback); anything else warns.""" - if error_code == "missing_scope": - logger.debug("Channel directory: Slack team %s lacks channels:read; using session history only", team_id) - else: - _warn_slack_directory(team_id, detail) - - # --- Build / refresh ------------------------------------------------------- async def build_channel_directory(adapters: Dict[Any, Any]) -> Dict[str, Any]: @@ -178,12 +172,10 @@ async def build_channel_directory(adapters: Dict[Any, Any]) -> Dict[str, Any]: for plat in Platform: await _discover(plat.value) # Plugin platforms are dynamic enum members missing from Platform.__members__. - try: + with contextlib.suppress(Exception): from gateway.platform_registry import platform_registry for entry in platform_registry.plugin_entries(): await _discover(entry.name) - except Exception: - pass _apply_channel_aliases(platforms) @@ -222,24 +214,18 @@ def _build_discord(adapter) -> List[Dict[str, str]]: def _slack_api_error_code(error: Exception) -> Optional[str]: - """Return Slack Web API error code from SlackApiError-like exceptions.""" - response = getattr(error, "response", None) - if response is None: - return None - try: - value = response.get("error") - except Exception: - return None - return str(value) if value else None + """Slack Web API error code from SlackApiError-like exceptions.""" + with contextlib.suppress(Exception): + value = error.response.get("error") + return str(value) if value else None + return None def _normalize_adapter_channels(raw_channels: Any) -> List[Dict[str, Any]]: """Validate and dedupe entries returned by an adapter's ``list_channels()`` hook.""" channels: List[Dict[str, Any]] = [] seen_ids = set() - if not isinstance(raw_channels, list): - return channels - for raw in raw_channels: + for raw in raw_channels if isinstance(raw_channels, list) else (): if not isinstance(raw, dict): continue channel_id = str(raw.get("id") or "").strip() @@ -260,47 +246,86 @@ def _slack_base_id(entry_id: str) -> str: return entry_id.split(":", 1)[0] -async def _build_slack(adapter) -> List[Dict[str, Any]]: - """List Slack channels the bot has joined across all workspaces. +def _slack_has_raw_name(entry: Dict[str, Any]) -> bool: + return entry.get("name", "").startswith(_SLACK_RAW_ID_PREFIXES) - ``users.conversations`` per workspace client (public + private member channels), - then merge DMs from session history. Missing channels:read falls back to - session history quietly rather than warning on every refresh. - """ + +async def _slack_team_channels(team_id: str, client, seen_ids: set) -> List[Dict[str, Any]]: + """``users.conversations`` for one workspace (public + private member channels), paginated.""" + channels: List[Dict[str, Any]] = [] + try: + cursor: Optional[str] = None + for _page in range(20): # safety cap on pagination + response = await client.users_conversations( + types="public_channel,private_channel", + exclude_archived=True, + limit=200, + cursor=cursor, + ) + if not response.get("ok"): + error_code = response.get("error", "unknown") + _report_slack_failure(team_id, error_code, f"users.conversations not ok: {error_code}") + break + for ch in response.get("channels", []): + cid, name = ch.get("id"), ch.get("name") + if not cid or not name or cid in seen_ids: + continue + seen_ids.add(cid) + channels.append({"id": cid, "name": name, "type": "private" if ch.get("is_private") else "channel"}) + cursor = (response.get("response_metadata") or {}).get("next_cursor") + if not cursor: + break + except Exception as e: + _report_slack_failure(team_id, _slack_api_error_code(e), str(e)) + return channels + + +async def _slack_resolve_raw_names(client, channels: List[Dict[str, Any]]) -> None: + """Name remaining raw-ID entries (DMs, channels outside bot scope) via + conversations.info + users.info once per base conversation, concurrently.""" + unresolved_by_base: Dict[str, list] = {} + for entry in channels: + if _slack_has_raw_name(entry): + unresolved_by_base.setdefault(_slack_base_id(entry["id"]), []).append(entry) + if not unresolved_by_base: + return + + async def _resolve_base(base_id: str, entries: list) -> None: + try: + resp = await client.conversations_info(channel=base_id) + if not resp.get("ok"): + return + ch_info = resp.get("channel", {}) + resolved_name = resolved_type = None + if not ch_info.get("is_im"): + resolved_name = ch_info.get("name") or ch_info.get("name_normalized") + elif ch_info.get("user", ""): + user_resp = await client.users_info(user=ch_info["user"]) + if user_resp.get("ok"): + u = user_resp["user"] + resolved_name = u.get("profile", {}).get("display_name") or u.get("real_name") or u.get("name") + resolved_type = "dm" + for entry in entries if resolved_name else (): + entry["name"] = resolved_name + if resolved_type: + entry["type"] = resolved_type + except Exception as e: + logger.debug("Channel directory: failed to resolve %s: %s", base_id, e) + + await asyncio.gather(*[_resolve_base(bid, ents) for bid, ents in unresolved_by_base.items()]) + + +async def _build_slack(adapter) -> List[Dict[str, Any]]: + """List Slack channels the bot has joined across all workspaces, merged with + session-history DMs. Missing channels:read falls back to session history quietly.""" team_clients = getattr(adapter, "_team_clients", None) or {} if not team_clients: return await asyncio.to_thread(_build_from_sessions, "slack") channels: List[Dict[str, Any]] = [] seen_ids: set = set() - for team_id, client in team_clients.items(): - try: - cursor: Optional[str] = None - for _page in range(20): # safety cap on pagination - response = await client.users_conversations( - types="public_channel,private_channel", - exclude_archived=True, - limit=200, - cursor=cursor, - ) - if not response.get("ok"): - error_code = response.get("error", "unknown") - _report_slack_failure(team_id, error_code, f"users.conversations not ok: {error_code}") - break - for ch in response.get("channels", []): - cid = ch.get("id") - name = ch.get("name") - if not cid or not name or cid in seen_ids: - continue - seen_ids.add(cid) - channels.append({"id": cid, "name": name, "type": "private" if ch.get("is_private") else "channel"}) - cursor = (response.get("response_metadata") or {}).get("next_cursor") - if not cursor: - break - except Exception as e: - _report_slack_failure(team_id, _slack_api_error_code(e), str(e)) - continue + channels.extend(await _slack_team_channels(team_id, client, seen_ids)) # Merge session-history DM/group entries, naming raw-ID entries from the # API-discovered channels where the base conversation ID is known. @@ -309,50 +334,12 @@ async def _build_slack(adapter) -> List[Dict[str, Any]]: eid = entry.get("id") if not isinstance(eid, str) or eid in seen_ids: continue - if entry.get("name", "").startswith(_SLACK_RAW_ID_PREFIXES): - base_id = _slack_base_id(eid) - if base_id in api_name_lookup: - entry["name"] = api_name_lookup[base_id] + if _slack_has_raw_name(entry) and _slack_base_id(eid) in api_name_lookup: + entry["name"] = api_name_lookup[_slack_base_id(eid)] channels.append(entry) seen_ids.add(eid) - # Resolve remaining raw-ID entries (DMs, channels outside bot scope) via - # conversations.info + users.info once per base conversation, concurrently. - unresolved = [ch for ch in channels if ch.get("name", "").startswith(_SLACK_RAW_ID_PREFIXES)] - if unresolved: - client = next(iter(team_clients.values())) - unresolved_by_base: Dict[str, list] = {} - for entry in unresolved: - unresolved_by_base.setdefault(_slack_base_id(entry["id"]), []).append(entry) - - async def _resolve_base(base_id: str, entries: list) -> None: - try: - resp = await client.conversations_info(channel=base_id) - if not resp.get("ok"): - return - ch_info = resp.get("channel", {}) - resolved_name = None - resolved_type = None - if ch_info.get("is_im"): - peer_user = ch_info.get("user", "") - if peer_user: - user_resp = await client.users_info(user=peer_user) - if user_resp.get("ok"): - u = user_resp["user"] - resolved_name = u.get("profile", {}).get("display_name") or u.get("real_name") or u.get("name") - resolved_type = "dm" - else: - resolved_name = ch_info.get("name") or ch_info.get("name_normalized") - if resolved_name: - for entry in entries: - entry["name"] = resolved_name - if resolved_type: - entry["type"] = resolved_type - except Exception as e: - logger.debug("Channel directory: failed to resolve %s: %s", base_id, e) - - await asyncio.gather(*[_resolve_base(bid, ents) for bid, ents in unresolved_by_base.items()]) - + await _slack_resolve_raw_names(next(iter(team_clients.values())), channels) return channels @@ -365,51 +352,49 @@ def _build_from_sessions(platform_name: str) -> List[Dict[str, str]]: return _build_from_sessions_db(platform_name) or _build_from_sessions_json(platform_name) -def _extend_from_origins(entries: List[Dict[str, Any]], origins: Iterable[Tuple[Dict[str, Any], Any]]) -> None: - """Append deduped directory entries for (origin, chat_type) pairs to *entries*. +def _entries_from_origins(platform_name: str, source: str, origins_fn) -> List[Dict[str, Any]]: + """Deduped directory entries for the (origin, chat_type) pairs yielded by ``origins_fn()``. Appends incrementally so a mid-iteration failure keeps the entries read so far. """ - seen_ids = set() - for origin, chat_type in origins: - entry_id = _session_entry_id(origin) - if not entry_id or entry_id in seen_ids: - continue - seen_ids.add(entry_id) - entries.append({ - "id": entry_id, "name": _session_entry_name(origin), - "type": chat_type, "thread_id": origin.get("thread_id"), - }) + entries: List[Dict[str, Any]] = [] + try: + seen_ids = set() + for origin, chat_type in origins_fn(): + entry_id = _session_entry_id(origin) + if not entry_id or entry_id in seen_ids: + continue + seen_ids.add(entry_id) + entries.append({ + "id": entry_id, "name": _session_entry_name(origin), + "type": chat_type, "thread_id": origin.get("thread_id"), + }) + except Exception as e: + logger.debug("Channel directory: %s for %s: %s", source, platform_name, e) + return entries def _build_from_sessions_db(platform_name: str) -> List[Dict[str, str]]: """Pull channels/contacts from state.db gateway session rows.""" - entries: List[Dict[str, Any]] = [] - try: + def _origins() -> Iterable[Tuple[Dict[str, Any], Any]]: from hermes_state import get_shared_session_db, release_or_close db = get_shared_session_db() try: lister = getattr(db, "list_gateway_sessions", None) if not callable(lister): - return [] + return rows = lister(platform=platform_name, active_only=False) finally: release_or_close(db) + for row in rows: + origin = None + with contextlib.suppress(TypeError, ValueError): + origin = json.loads(row["origin_json"]) if row.get("origin_json") else None + if not isinstance(origin, dict) or not origin: + origin = {"chat_id": row.get("chat_id"), "thread_id": row.get("thread_id"), "chat_name": row.get("display_name")} + yield origin, row.get("chat_type") or "dm" - def _origin(row) -> Dict[str, Any]: - if row.get("origin_json"): - try: - parsed = json.loads(row["origin_json"]) - if isinstance(parsed, dict) and parsed: - return parsed - except (TypeError, ValueError): - pass - return {"chat_id": row.get("chat_id"), "thread_id": row.get("thread_id"), "chat_name": row.get("display_name")} - - _extend_from_origins(entries, ((_origin(row), row.get("chat_type") or "dm") for row in rows)) - except Exception as e: - logger.debug("Channel directory: state.db session read failed for %s: %s", platform_name, e) - return entries + return _entries_from_origins(platform_name, "state.db session read failed", _origins) def _build_from_sessions_json(platform_name: str) -> List[Dict[str, str]]: @@ -417,25 +402,19 @@ def _build_from_sessions_json(platform_name: str) -> List[Dict[str, str]]: sessions_path = get_hermes_home() / "sessions" / "sessions.json" if not sessions_path.exists(): return [] - entries: List[Dict[str, Any]] = [] - try: + + def _origins() -> Iterable[Tuple[Dict[str, Any], Any]]: with open(sessions_path, encoding="utf-8") as f: data = json.load(f) + for _key, session in data.items(): + # Keys starting with "_" (e.g. the gateway's "_README") are metadata sentinels. + if str(_key).startswith("_") or not isinstance(session, dict): + continue + origin = session.get("origin") or {} + if origin.get("platform") == platform_name: + yield origin, session.get("chat_type", "dm") - def _origins(): - for _key, session in data.items(): - # Keys starting with "_" (e.g. the gateway's "_README") are - # metadata sentinels, not sessions. - if str(_key).startswith("_") or not isinstance(session, dict): - continue - origin = session.get("origin") or {} - if origin.get("platform") == platform_name: - yield origin, session.get("chat_type", "dm") - - _extend_from_origins(entries, _origins()) - except Exception as e: - logger.debug("Channel directory: failed to read sessions for %s: %s", platform_name, e) - return entries + return _entries_from_origins(platform_name, "failed to read sessions", _origins) # --- Read / resolve -------------------------------------------------------- @@ -444,21 +423,19 @@ def load_directory() -> Dict[str, Any]: """Load the cached directory from disk, with aliases re-applied on read.""" directory_path = _directory_path() if directory_path.exists(): - try: + with contextlib.suppress(Exception): with open(directory_path, encoding="utf-8") as f: data = json.load(f) # Aliases apply on read too, so new names take effect between timed rebuilds. _apply_channel_aliases(data.setdefault("platforms", {})) return data - except Exception: - pass base = {"updated_at": None, "platforms": {}} _apply_channel_aliases(base["platforms"]) return base def lookup_channel_type(platform_name: str, chat_id: str) -> Optional[str]: - """Return the channel ``type`` string (e.g. ``"channel"``, ``"forum"``) for *chat_id*, or *None* if unknown.""" + """Channel ``type`` string (e.g. ``"channel"``, ``"forum"``) for *chat_id*, or None if unknown.""" for ch in load_directory().get("platforms", {}).get(platform_name, []): if ch.get("id") == chat_id: return ch.get("type") @@ -487,9 +464,7 @@ def resolve_channel_name(platform_name: str, name: str) -> Optional[str]: # 1. Exact name match, including the display labels shown by send_message(action="list") for ch in channels: - if _normalize_channel_query(ch["name"]) == query: - return ch["id"] - if _normalize_channel_query(_channel_target_name(platform_name, ch)) == query: + if query in (_normalize_channel_query(ch["name"]), _normalize_channel_query(_channel_target_name(platform_name, ch))): return ch["id"] # 2. Guild-qualified match for Discord ("GuildName/channel") @@ -531,22 +506,17 @@ def format_directory_for_display(platforms: Optional[Dict[str, Any]] = None) -> f"{plat_name}:, or bare '{plat_name}' for the home channel)" ) elif plat_name == "discord": - # Group Discord channels by guild; DMs last. + # Group Discord channels by guild (sorted by name); DMs last, in discovery order. guilds: Dict[str, List] = {} dms: List = [] for ch in channels: - guild = ch.get("guild") - (guilds.setdefault(guild, []) if guild else dms).append(ch) - - for guild_name, guild_channels in sorted(guilds.items()): - lines.append(f"Discord ({guild_name}):") - lines.extend( - f" discord:{_channel_target_name(plat_name, ch)}" - for ch in sorted(guild_channels, key=lambda c: c["name"]) - ) + (guilds.setdefault(ch["guild"], []) if ch.get("guild") else dms).append(ch) + groups = [(f"Discord ({g}):", sorted(chs, key=lambda c: c["name"])) for g, chs in sorted(guilds.items())] if dms: - lines.append("Discord (DMs):") - lines.extend(f" discord:{_channel_target_name(plat_name, ch)}" for ch in dms) + groups.append(("Discord (DMs):", dms)) + for header, group in groups: + lines.append(header) + lines.extend(f" discord:{_channel_target_name(plat_name, ch)}" for ch in group) else: lines.append(f"{plat_name.title()}:") lines.extend(f" {plat_name}:{_channel_target_name(plat_name, ch)}" for ch in channels) diff --git a/gateway/code_skew.py b/gateway/code_skew.py index 9b77237c3a..a5a200142f 100644 --- a/gateway/code_skew.py +++ b/gateway/code_skew.py @@ -1,15 +1,11 @@ """Detect when the gateway is running stale code after a hot ``git pull``. The gateway's ``sys.modules`` is frozen at boot. If the checkout is updated -underneath it (manual ``git pull``, or the window before ``hermes update``'s -graceful restart), a first-time lazy import can resolve a freshly-pulled module -against a stale cached dependency -> ImportError -(``tests/test_stale_utils_module_import.py``). We snapshot the revision at +underneath it, a first-time lazy import can resolve a freshly-pulled module +against a stale cached dependency -> ImportError. We snapshot the revision at startup so risky callers (e.g. ``/model`` switching) can refuse with a clear -"restart the gateway" message instead. - -If the revision can't be read (non-git install, IO error) the boot snapshot -stays ``None`` and detection no-ops — never a false positive. +"restart the gateway" message. If the revision can't be read (non-git install, +IO error) the boot snapshot stays ``None`` and detection no-ops — never a false positive. """ from __future__ import annotations @@ -41,17 +37,12 @@ def record_boot_fingerprint() -> None: def _short(fingerprint: str) -> str: """Render a ``git::`` fingerprint as a compact label.""" sha = fingerprint.rsplit(":", 1)[-1] - if sha and sha != "unresolved" and len(sha) > 10: - return sha[:10] - return sha or fingerprint + return sha[:10] if sha and sha != "unresolved" and len(sha) > 10 else (sha or fingerprint) def detect_code_skew() -> tuple[str, str] | None: - """Return ``(boot_rev, disk_rev)`` short labels if the checkout drifted - since boot, else ``None``.""" - if _boot_fingerprint is None: - return None - current = _fingerprint() + """``(boot_rev, disk_rev)`` short labels if the checkout drifted since boot, else ``None``.""" + current = _fingerprint() if _boot_fingerprint is not None else None if current is None or current == _boot_fingerprint: return None return _short(_boot_fingerprint), _short(current) diff --git a/gateway/config.py b/gateway/config.py index 415450d1df..c428da7be5 100644 --- a/gateway/config.py +++ b/gateway/config.py @@ -1,18 +1,13 @@ -""" -Gateway configuration management. - -Handles loading and validating configuration for: -- Connected platforms (Telegram, Discord, WhatsApp, Weixin, and more) -- Home channels for each platform -- Session reset policies -- Delivery preferences +"""Gateway configuration: connected platforms, home channels, session reset +policies and delivery preferences, loaded from config.yaml / gateway.json / env. """ +import contextlib import logging import math import os from pathlib import Path -from dataclasses import asdict, dataclass, field, is_dataclass +from dataclasses import asdict, dataclass, field, fields, is_dataclass from typing import Dict, List, Optional, Any, Callable from enum import Enum @@ -27,6 +22,9 @@ from utils import is_truthy_value logger = logging.getLogger(__name__) +_TRUTHY_STRINGS = frozenset({"1", "true", "yes", "on"}) +_FALSY_STRINGS = frozenset({"0", "false", "no", "off"}) + def _coerce_bool(value: Any, default: bool = True) -> bool: """Coerce bool-ish config values, preserving a caller-provided default.""" @@ -34,21 +32,17 @@ def _coerce_bool(value: Any, default: bool = True) -> bool: return default if isinstance(value, str): lowered = value.strip().lower() - if lowered in {"true", "1", "yes", "on"}: + if lowered in _TRUTHY_STRINGS: return True - if lowered in {"false", "0", "no", "off"}: + if lowered in _FALSY_STRINGS: return False return default return is_truthy_value(value, default=default) def _normalize_multiplex_profile_allowlist(value: Any) -> Optional[List[str]]: - """Normalize the optional named-profile allowlist. - - ``None`` preserves the historical serve-all behavior. A malformed outer - value fails safe to an empty list (default profile only); malformed list - entries are skipped with a warning. - """ + """Normalize the optional named-profile allowlist: ``None`` = serve all; a malformed + outer value fails safe to ``[]`` (default profile only); bad entries are skipped.""" if value is None: return None if not isinstance(value, list): @@ -64,10 +58,7 @@ def _normalize_multiplex_profile_allowlist(value: Any) -> Optional[List[str]]: normalized: List[str] = [] for entry in value: if not isinstance(entry, str): - logger.warning( - "Skipping invalid gateway.multiplex_profile_allowlist entry %r (expected a profile name)", - entry, - ) + logger.warning("Skipping invalid gateway.multiplex_profile_allowlist entry %r (expected a profile name)", entry) continue try: name = normalize_profile_name(entry) @@ -80,51 +71,34 @@ def _normalize_multiplex_profile_allowlist(value: Any) -> Optional[List[str]]: return normalized -# Recognized truthy / falsy tokens for the GATEWAY_MULTIPLEX_PROFILES operator -# override. Anything not in either set — and a blank/whitespace value — is -# treated as "unset" so it falls through to config.yaml rather than silently -# forcing the flag off. -_MULTIPLEX_TRUTHY_STRINGS = frozenset({"1", "true", "yes", "on"}) -_MULTIPLEX_FALSY_STRINGS = frozenset({"0", "false", "no", "off"}) - - def _env_multiplex_profiles_override() -> "bool | None": - """Resolve the GATEWAY_MULTIPLEX_PROFILES operator override. + """GATEWAY_MULTIPLEX_PROFILES operator override: True/False for a recognized token. - Returns ``True``/``False`` when the env var is set to a recognized truthy/ - falsy token, or ``None`` when it is unset, blank, or unrecognized — in which - case the caller keeps the config.yaml value (env > config > default). Blank - is deliberately ``None``, not ``False``: a provisioned-but-unpopulated Fly - secret arrives as ``""`` and must NOT shadow a config.yaml opt-in. + ``None`` when unset, blank, or unrecognized so the caller keeps the config.yaml + value (env > config > default). Blank is deliberately ``None``, not ``False``: + a provisioned-but-unpopulated Fly secret arrives as ``""`` and must NOT shadow + a config.yaml opt-in. """ raw = os.getenv("GATEWAY_MULTIPLEX_PROFILES") token = (raw or "").strip().lower() if not token: return None - if token in _MULTIPLEX_TRUTHY_STRINGS: + if token in _TRUTHY_STRINGS: return True - if token in _MULTIPLEX_FALSY_STRINGS: + if token in _FALSY_STRINGS: return False logger.warning( "Ignoring unrecognized GATEWAY_MULTIPLEX_PROFILES=%r " "(expected one of %s or %s); falling back to config.yaml.", - raw, - sorted(_MULTIPLEX_TRUTHY_STRINGS), - sorted(_MULTIPLEX_FALSY_STRINGS), + raw, sorted(_TRUTHY_STRINGS), sorted(_FALSY_STRINGS), ) return None def _normalize_transport_token(value: Any) -> str: - """Normalize a streaming transport/mode value to a canonical token. - - Handles the YAML 1.1 boolean quirk where bare ``on`` / ``off`` parse to - Python ``True`` / ``False`` (see ``gateway/display_config.py`` ``_normalise``). - Without this, ``mode: off`` arrives as boolean ``False`` and stringifying it - yields ``"false"`` instead of the advertised ``"off"``, so streaming would be - enabled instead of disabled. Booleans map to ``"auto"`` (True) / ``"off"`` - (False); anything else is lower-cased, defaulting to ``"auto"``. - """ + """Canonical streaming transport token. YAML 1.1 parses bare ``on``/``off`` as + booleans (``mode: off`` → ``False`` → ``"false"`` would ENABLE streaming), so + booleans map to ``"auto"``/``"off"``; anything else lower-cases, default ``"auto"``.""" if value is None: return "auto" if isinstance(value, bool): @@ -133,50 +107,30 @@ def _normalize_transport_token(value: Any) -> str: def _coerce_float(value: Any, default: float) -> float: - """Coerce numeric config values, falling back on malformed input.""" - if value is None: - return default try: - return float(value) + return default if value is None else float(value) except (TypeError, ValueError): return default def _coerce_int(value: Any, default: int) -> int: - """Coerce integer config values, falling back on malformed input.""" - if value is None: - return default + # OverflowError: ``int(float("inf"))`` — non-finite YAML must degrade, not abort loading. try: - return int(value) + return default if value is None else int(value) except (TypeError, ValueError, OverflowError): - # OverflowError: int(float("inf")) — a non-finite YAML value must - # degrade to the default, not abort gateway config loading. return default def _coerce_optional_positive_int(value: Any, key: str) -> Optional[int]: - """Coerce an optional positive integer config value. - - ``None``/0/negative disable the setting. Malformed values are ignored with - a warning so a typo never prevents the gateway from starting. - """ + """``None``/0/negative disable; malformed values are ignored with a warning so a typo never blocks startup.""" if value is None: return None try: - if isinstance(value, bool): + if isinstance(value, bool) or (isinstance(value, float) and not value.is_integer()): raise ValueError(value) - if isinstance(value, float): - if not value.is_integer(): - raise ValueError(value) - parsed = int(value) - elif isinstance(value, str): - parsed = int(value.strip(), 10) - else: - parsed = int(value) + parsed = int(value.strip(), 10) if isinstance(value, str) else int(value) except (TypeError, ValueError): - logger.warning( - "Ignoring invalid %s=%r (expected a positive integer; 0/null disables)", key, value - ) + logger.warning("Ignoring invalid %s=%r (expected a positive integer; 0/null disables)", key, value) return None return parsed if parsed > 0 else None @@ -187,11 +141,8 @@ _SYSTEMD_WATCHDOG_MAX_SECONDS = 2_147_483_647 def coerce_systemd_watchdog_seconds( value: Any, key: str = "gateway.systemd_watchdog_seconds" ) -> int: - """Return a bounded positive watchdog interval or zero when disabled. - - Runtime and service generation share this normalization so a value can - never enable ``Type=notify`` while disabling application heartbeats. - """ + """Bounded positive watchdog interval, or zero when disabled/invalid. Shared by runtime + and service generation so a value can never enable ``Type=notify`` without heartbeats.""" if value is None: return 0 parsed: Optional[int] = None @@ -207,20 +158,13 @@ def coerce_systemd_watchdog_seconds( if parsed is None: logger.warning("Ignoring invalid %s (expected a positive integer)", key) return 0 - if parsed == 0: - return 0 - if not 0 < parsed <= _SYSTEMD_WATCHDOG_MAX_SECONDS: - logger.warning( - "Ignoring invalid %s (expected an integer from 1 to %d)", - key, - _SYSTEMD_WATCHDOG_MAX_SECONDS, - ) + if parsed and not 0 < parsed <= _SYSTEMD_WATCHDOG_MAX_SECONDS: + logger.warning("Ignoring invalid %s (expected an integer from 1 to %d)", key, _SYSTEMD_WATCHDOG_MAX_SECONDS) return 0 return parsed def _coerce_dict(value: Any) -> Dict[str, Any]: - """Return *value* when it is a mapping, otherwise an empty dict.""" return value if isinstance(value, dict) else {} @@ -243,14 +187,8 @@ def _dict_slot(container: dict, key: str) -> dict: def _getenv(name: str, default: Optional[str] = None) -> Optional[str]: - """Read env vars through the active profile secret scope when present. - - ``load_gateway_config()`` runs in many contexts, including multiplexed - profile startup where ``_profile_runtime_scope`` installs per-profile - secrets. In that scope we must prefer the scoped value; outside it we keep - legacy ``os.getenv`` behavior for single-profile callers and unscoped - gateway reads. - """ + """Env read through the active profile secret scope when present (multiplexed + per-profile secrets must win); otherwise legacy ``os.getenv``.""" if current_secret_scope() is not None: scope_val = _get_secret(name, None) return scope_val if scope_val is not None else default @@ -262,19 +200,13 @@ def _getenv_str(name: str, default: str = "") -> str: return val if val is not None else default -# Module-level cache for bundled platform plugin names (lives outside the -# enum so it doesn't become an accidental enum member). +# Bundled platform plugin names, cached outside the enum so it never becomes a member. _Platform__bundled_plugin_names: Optional[set] = None class Platform(Enum): - """Supported messaging platforms. - - Built-in platforms have explicit members. Plugin platforms use dynamic - members created on-demand by ``_missing_()`` so that - ``Platform("irc")`` works without modifying this enum. Dynamic members - are cached in ``_value2member_map_`` for identity-stable comparisons. - """ + """Supported messaging platforms. Plugin platforms are dynamic members created on + demand by ``_missing_`` and cached so ``Platform("irc") is Platform("irc")`` holds.""" LOCAL = "local" TELEGRAM = "telegram" DISCORD = "discord" @@ -299,39 +231,25 @@ class Platform(Enum): QQBOT = "qqbot" YUANBAO = "yuanbao" RELAY = "relay" # generic relay adapter fronted by the connector (EXPERIMENTAL) + @classmethod def _missing_(cls, value): - """Accept unknown platform names only for known plugin adapters. - - Creates a pseudo-member cached in ``_value2member_map_`` so that - ``Platform("irc") is Platform("irc")`` holds True (identity-stable). - Arbitrary strings are rejected to prevent enum pollution. - """ + """Accept unknown names only for bundled or runtime-registered plugin adapters (no enum pollution).""" if not isinstance(value, str) or not value.strip(): return None - # Normalise to lowercase to avoid case mismatches in config value = value.strip().lower() - # Check cache first (another call may have created it already) if value in cls._value2member_map_: return cls._value2member_map_[value] - # Only create pseudo-members for bundled plugin platforms (discovered - # via filesystem scan) or runtime-registered plugin platforms. global _Platform__bundled_plugin_names if _Platform__bundled_plugin_names is None: _Platform__bundled_plugin_names = cls._scan_bundled_plugin_platforms() if value in _Platform__bundled_plugin_names: return cls._add_pseudo_member(value) - - # Runtime-registered plugins (e.g. user-installed, discovered after - # the enum was defined). - try: + with contextlib.suppress(Exception): from gateway.platform_registry import platform_registry if platform_registry.is_registered(value): return cls._add_pseudo_member(value) - except Exception: - pass - return None @classmethod @@ -345,62 +263,39 @@ class Platform(Enum): @classmethod def _scan_bundled_plugin_platforms(cls) -> set: - """Return names of bundled platform plugins under ``plugins/platforms/``.""" - names: set = set() + """Names of bundled platform plugins under ``plugins/platforms/``.""" try: platforms_dir = Path(__file__).parent.parent / "plugins" / "platforms" - if platforms_dir.is_dir(): - for child in platforms_dir.iterdir(): - if child.is_dir() and (child / "__init__.py").exists() and ( - (child / "plugin.yaml").exists() or (child / "plugin.yml").exists() - ): - names.add(child.name.lower()) + return { + child.name.lower() + for child in (platforms_dir.iterdir() if platforms_dir.is_dir() else ()) + if child.is_dir() and (child / "__init__.py").exists() + and ((child / "plugin.yaml").exists() or (child / "plugin.yml").exists()) + } except Exception: - pass - return names + return set() -# Snapshot of built-in platform values before any dynamic _missing_ lookups. -# Used to distinguish real platforms from arbitrary strings. +# Built-in values snapshotted before any dynamic _missing_ lookup. _BUILTIN_PLATFORM_VALUES = frozenset(m.value for m in Platform.__members__.values()) -# Platforms that bind a host TCP port (HTTP/webhook listeners). In a profile -# multiplexer the default profile owns the single shared listener and serves -# every profile through the /p// URL prefix, so a SECONDARY profile -# enabling one of these is always a misconfiguration: it would try to bind a -# port already held by the default's listener. Single source of truth for -# both the gateway's fail-fast startup validation (gateway/run.py) and the -# dashboard's pre-write mutation validation (hermes_cli/web_server.py) so -# the two policies cannot drift. Stored as platform .value strings. +# Platforms that bind a host TCP port. In a multiplexer only the default profile owns +# the shared listener, so a SECONDARY profile enabling one is a misconfiguration. +# Single source of truth for gateway/run.py and hermes_cli/web_server.py validation. PORT_BINDING_PLATFORM_VALUES = frozenset({ - "webhook", - "api_server", - "msgraph_webhook", - "feishu", - "wecom_callback", - "bluebubbles", - "sms", - "whatsapp_cloud", - "line", - "teams", + "webhook", "api_server", "msgraph_webhook", "feishu", "wecom_callback", + "bluebubbles", "sms", "whatsapp_cloud", "line", "teams", }) -# Platforms whose port-binding status depends on connection mode. Feishu in -# websocket mode (its default) uses an outbound long connection — no listener. -# Only webhook/callback mode binds a port. Maps platform value → the mode -# value that actually binds (#52563). +# Platforms that only bind in one connection mode (Feishu's default websocket mode is outbound). PORT_BINDING_CONDITIONAL_MODES: dict[str, str] = { "feishu": "webhook", } def platform_binds_port(platform_value: str, extra: Optional[dict] = None) -> bool: - """Return True when *platform_value* actually binds a port for *extra* config. - - Mode-conditional platforms (Feishu) only bind in their listener mode; - everything else in ``PORT_BINDING_PLATFORM_VALUES`` always binds. - """ + """True when *platform_value* actually binds a port for *extra* config.""" if platform_value not in PORT_BINDING_PLATFORM_VALUES: return False expected_mode = PORT_BINDING_CONDITIONAL_MODES.get(platform_value) @@ -412,45 +307,26 @@ def platform_binds_port(platform_value: str, extra: Optional[dict] = None) -> bo @dataclass class HomeChannel: - """ - Default destination for a platform. - - When a cron job specifies deliver="telegram" without a specific chat ID, - messages are sent to this home channel. Thread-aware platforms may also - store a thread/topic ID so the bare platform target routes to the exact - conversation where /sethome was run. - """ + """Default destination for a platform (``deliver="telegram"`` without a chat ID); + ``thread_id`` routes the bare target to the topic where /sethome was run.""" platform: Platform chat_id: str - name: str # Human-readable name for display + name: str thread_id: Optional[str] = None - # Authenticated logical-target provenance observed by a platform adapter. - # Relay egress re-attaches these values, but the connector remains the - # authorization boundary and resolves them against its authoritative stores. + # Authenticated logical-target provenance; relay egress re-attaches these but the + # connector remains the authorization boundary. user_id: Optional[str] = None scope_id: Optional[str] = None - + def to_dict(self) -> Dict[str, Any]: - result = { - "platform": self.platform.value, - "chat_id": self.chat_id, - "name": self.name, - } - for key in ("thread_id", "user_id", "scope_id"): - if getattr(self, key): - result[key] = getattr(self, key) + result = {"platform": self.platform.value, "chat_id": self.chat_id, "name": self.name} + result.update({k: v for k in ("thread_id", "user_id", "scope_id") if (v := getattr(self, k))}) return result - + @classmethod def from_dict(cls, data: Dict[str, Any]) -> "HomeChannel": - return cls( - platform=Platform(data["platform"]), - chat_id=str(data["chat_id"]), - name=data.get("name", "Home"), - thread_id=str(data["thread_id"]) if data.get("thread_id") else None, - user_id=str(data["user_id"]) if data.get("user_id") else None, - scope_id=str(data["scope_id"]) if data.get("scope_id") else None, - ) + optional = {k: str(data[k]) if data.get(k) else None for k in ("thread_id", "user_id", "scope_id")} + return cls(platform=Platform(data["platform"]), chat_id=str(data["chat_id"]), name=data.get("name", "Home"), **optional) def persist_home_channel(home: HomeChannel, *, enabled_if_new: bool = False) -> None: @@ -467,71 +343,39 @@ def persist_home_channel(home: HomeChannel, *, enabled_if_new: bool = False) -> @dataclass class SessionResetPolicy: - """ - Controls when sessions reset (lose context). - - Modes: - - "daily": Reset at a specific hour each day - - "idle": Reset after N minutes of inactivity - - "both": Whichever triggers first (daily boundary OR idle timeout) - - "none": Never auto-reset (context managed only by compression) - - Default is "none" — sessions never auto-reset unless the user opts in - via the `session_reset` section in config.yaml (or gateway.json - overrides). Changed July 2026 from "both" (24h idle + daily 4am), which - surprised users who expected their conversations to persist. - """ - mode: str = "none" # "daily", "idle", "both", or "none" - at_hour: int = 4 # Hour for daily reset (0-23, local time) - idle_minutes: int = 1440 # Minutes of inactivity before reset (24 hours) - notify: bool = True # Send a notification to the user when auto-reset occurs - notify_exclude_platforms: tuple = ("api_server", "webhook") # Platforms that don't get reset notifications - # A background process this many hours old (or older) no longer blocks - # session idle/daily reset. A forgotten preview server should not keep a - # session alive forever (#29177). The process is NOT killed — only ignored - # by the reset guard. Raise this if you run legitimate multi-day jobs whose - # liveness should pin the conversation open. + """When sessions reset: "daily" (at ``at_hour``), "idle" (after ``idle_minutes``), + "both" (whichever first), "none" (default: only compression manages context).""" + mode: str = "none" + at_hour: int = 4 # 0-23, local time + idle_minutes: int = 1440 + notify: bool = True # Notify the user when auto-reset occurs + notify_exclude_platforms: tuple = ("api_server", "webhook") + # A background process this old no longer blocks reset (a forgotten preview server + # must not pin a session forever); it is NOT killed, only ignored by the guard. bg_process_max_age_hours: int = 24 def to_dict(self) -> Dict[str, Any]: - return { - "mode": self.mode, - "at_hour": self.at_hour, - "idle_minutes": self.idle_minutes, - "notify": self.notify, - "notify_exclude_platforms": list(self.notify_exclude_platforms), - "bg_process_max_age_hours": self.bg_process_max_age_hours, - } - + return {**asdict(self), "notify_exclude_platforms": list(self.notify_exclude_platforms)} + @classmethod def from_dict(cls, data: Dict[str, Any]) -> "SessionResetPolicy": data = _coerce_dict(data) - - def val(key: str, default: Any) -> Any: - # Handle both missing keys and explicit null values (YAML null → None) - value = data.get(key) - return default if value is None else value - exclude = data.get("notify_exclude_platforms") + # Missing keys and explicit YAML nulls both take the field default. + plain = { + f.name: f.default if data.get(f.name) is None else data[f.name] + for f in fields(cls) if f.name not in ("notify", "notify_exclude_platforms") + } return cls( - mode=val("mode", "none"), - at_hour=val("at_hour", 4), - idle_minutes=val("idle_minutes", 1440), notify=_coerce_bool(data.get("notify"), True), notify_exclude_platforms=tuple(exclude) if exclude is not None else ("api_server", "webhook"), - bg_process_max_age_hours=val("bg_process_max_age_hours", 24), + **plain, ) @dataclass class ChannelOverride: - """ - Per-channel override for model, provider, and system prompt. - - Used in config under platforms..channel_overrides[channel_id]. - Enables different channels (e.g. Discord #daily vs #dev) to use different - models and personas without running separate gateway instances. - """ + """Per-channel model/provider/system_prompt override (``platforms..channel_overrides[channel_id]``).""" model: Optional[str] = None provider: Optional[str] = None system_prompt: Optional[str] = None @@ -543,19 +387,12 @@ class ChannelOverride: def from_dict(cls, data: Dict[str, Any]) -> "ChannelOverride": if not data: return cls() - return cls( - model=data.get("model"), - provider=data.get("provider"), - system_prompt=data.get("system_prompt"), - ) + return cls(model=data.get("model"), provider=data.get("provider"), system_prompt=data.get("system_prompt")) -# Canonical map of platforms whose primary credential is ``PlatformConfig.token`` -# and the env var it loads from. Used for empty-token warnings at config -# validation and by the multiplex primary-startup credential gate in -# ``gateway.run`` (#64674). Platforms absent from this map authenticate some -# other way (session files, port-bound webhooks, api_key-only) and must never -# be skipped for a missing token. +# Platforms whose primary credential is ``PlatformConfig.token`` → its env var (empty-token +# warnings; multiplex primary-startup gate in ``gateway.run``). Platforms absent here +# authenticate another way and must never be skipped for a missing token. PLATFORM_TOKEN_ENV_NAMES: dict["Platform", str] = { Platform.TELEGRAM: "TELEGRAM_BOT_TOKEN", Platform.DISCORD: "DISCORD_BOT_TOKEN", @@ -570,67 +407,35 @@ PLATFORM_TOKEN_ENV_NAMES: dict["Platform", str] = { class PlatformConfig: """Configuration for a single messaging platform.""" enabled: bool = False - token: Optional[str] = None # Bot token (Telegram, Discord) + token: Optional[str] = None api_key: Optional[str] = None # API key if different from token home_channel: Optional[HomeChannel] = None - - # Reply threading mode (Telegram/Slack) - # - "off": Never thread replies to original message - # - "first": Only first chunk threads to user's message (default) - # - "all": All chunks in multi-part replies thread to user's message + # Reply threading: "off" never threads, "first" threads only the first chunk, "all" every chunk. reply_to_mode: str = "first" - - # Whether the gateway is allowed to send "♻️ Gateway online" / - # "♻ Gateway restarted" lifecycle notifications on this platform. - # Default True preserves prior behavior. Set False on platforms used - # by end users (e.g. Slack) where operator-flavored restart pings are - # noise; keep True for back-channels where the operator wants them. + # "♻️ Gateway online/restarted" pings; False on end-user platforms where they are noise. gateway_restart_notification: bool = True - - # Whether the gateway shows a "typing…" / "is thinking…" status indicator - # while the agent processes a message on this platform. Default True - # preserves prior behavior. Set False on platforms where the indicator is - # unwanted (e.g. Slack's assistant.threads.setStatus "is thinking…", which - # disables the compose box, or any platform where users find the bubble - # noisy). Drives the per-message _keep_typing refresh loop in - # gateway/platforms/base.py. + # "typing…" indicator (drives _keep_typing in platforms/base.py); False where unwanted + # (Slack's setStatus disables the compose box). typing_indicator: bool = True - - # Custom text for the working-state line on platforms whose typing - # indicator renders text rather than a native bubble: Slack's - # assistant.threads.setStatus line (shown next to the bot name; needs the - # assistant:write scope to render) and Google Chat's visible marker - # message. None keeps each platform's built-in default ("is thinking..." / - # "Hermes is thinking…"). Platforms with textless indicators (Discord, - # Telegram, Matrix, …) ignore it. + # Working-state text for text-rendering indicators (Slack assistant status, Google + # Chat marker); None = platform default; textless indicators ignore it. typing_status_text: Optional[str] = None - - # Per-channel model/provider/system_prompt overrides (channel_id -> ChannelOverride) channel_overrides: Dict[str, ChannelOverride] = field(default_factory=dict) - - # Platform-specific settings - extra: Dict[str, Any] = field(default_factory=dict) + extra: Dict[str, Any] = field(default_factory=dict) # Platform-specific settings def to_dict(self) -> Dict[str, Any]: result = { - "enabled": self.enabled, - "extra": self.extra, - "reply_to_mode": self.reply_to_mode, + "enabled": self.enabled, "extra": self.extra, "reply_to_mode": self.reply_to_mode, "gateway_restart_notification": self.gateway_restart_notification, "typing_indicator": self.typing_indicator, } if self.typing_status_text is not None: result["typing_status_text"] = self.typing_status_text - if self.token: - result["token"] = self.token - if self.api_key: - result["api_key"] = self.api_key + result.update({k: v for k in ("token", "api_key") if (v := getattr(self, k))}) if self.home_channel: result["home_channel"] = self.home_channel.to_dict() if self.channel_overrides: - result["channel_overrides"] = { - cid: ov.to_dict() for cid, ov in self.channel_overrides.items() - } + result["channel_overrides"] = {cid: ov.to_dict() for cid, ov in self.channel_overrides.items()} return result @classmethod @@ -640,22 +445,19 @@ class PlatformConfig: if isinstance(data.get("home_channel"), dict): home_channel = HomeChannel.from_dict(data["home_channel"]) - # gateway_restart_notification / typing_indicator / typing_status_text may - # arrive top-level or bridged into ``extra`` by the shared-key loop in - # load_gateway_config(), so YAML ``discord: gateway_restart_notification: false`` - # works without a separate platforms: block. Check both (top-level wins). + # The typing/restart-notification keys may be top-level or bridged into ``extra``; top-level wins. extra = _coerce_dict(data.get("extra", {})) def toplevel_or_extra(key: str) -> Any: value = data.get(key) return extra.get(key) if value is None else value - channel_overrides: Dict[str, ChannelOverride] = {} raw_overrides = data.get("channel_overrides") or {} - if isinstance(raw_overrides, dict): - for cid, ov_data in raw_overrides.items(): - if isinstance(ov_data, dict): - channel_overrides[str(cid)] = ChannelOverride.from_dict(ov_data) + channel_overrides = { + str(cid): ChannelOverride.from_dict(ov_data) + for cid, ov_data in raw_overrides.items() + if isinstance(ov_data, dict) + } if isinstance(raw_overrides, dict) else {} return cls( enabled=_coerce_bool(data.get("enabled"), False), @@ -671,11 +473,8 @@ class PlatformConfig: ) -# Streaming defaults — single source of truth so both StreamingConfig and -# StreamConsumerConfig agree on the out-of-the-box edit rhythm. Tuned for -# Telegram's ~1 edit/s flood envelope: a touch under 1s lets the cadence -# breathe without bumping into rate limits, and a smaller buffer threshold -# makes short replies feel near-instant in DMs. +# Shared by StreamingConfig and StreamConsumerConfig. Tuned for Telegram's ~1 edit/s +# flood envelope; the small buffer threshold makes short DM replies feel instant. DEFAULT_STREAMING_EDIT_INTERVAL: float = 0.8 DEFAULT_STREAMING_BUFFER_THRESHOLD: int = 24 DEFAULT_STREAMING_CURSOR: str = " ▉" @@ -683,79 +482,37 @@ DEFAULT_STREAMING_CURSOR: str = " ▉" @dataclass class StreamingConfig: - """Configuration for real-time token streaming to messaging platforms.""" + """Real-time token streaming to messaging platforms.""" enabled: bool = False - # Transport selection: - # "auto" — prefer native streaming-draft updates when the platform - # supports them (Telegram sendMessageDraft, Bot API 9.5+); - # fall back to edit-based when not. - # "draft" — explicitly request native drafts; falls back to edit when - # the platform/chat doesn't support them. - # "edit" — progressive editMessageText only (legacy behaviour). - # "off" — disable streaming entirely. - # - # Default is "auto": prefer native draft streaming on platforms that - # support it (Telegram DMs via sendMessageDraft, Bot API 9.5+) and fall - # back to edit-based streaming everywhere else. This is safe as a global - # default because adapters without draft support (Discord, Slack, Matrix, - # …) report supports_draft_streaming() == False and transparently use the - # edit path — so "auto" never regresses non-Telegram platforms, it only - # upgrades the chats that can render the smoother native preview. + # "auto" prefers native drafts (Telegram sendMessageDraft) with edit fallback — safe + # globally since adapters without draft support use the edit path unchanged; + # "draft" / "edit" force one; "off" disables. transport: str = "auto" edit_interval: float = DEFAULT_STREAMING_EDIT_INTERVAL buffer_threshold: int = DEFAULT_STREAMING_BUFFER_THRESHOLD cursor: str = DEFAULT_STREAMING_CURSOR - # Ported from openclaw/openclaw#72038. When >0, the final edit for - # a long-running streamed response is delivered as a fresh message - # if the original preview has been visible for at least this many - # seconds, so the platform's visible timestamp reflects completion - # time instead of the preview creation time. Currently applied to - # Telegram only (other platforms ignore the setting). Default 0 disables - # the fresh-message replacement path; set >0 to opt in. + # >0: deliver the final edit as a fresh message when the preview has been visible + # this long, so the timestamp reflects completion. Telegram only; 0 disables. fresh_final_after_seconds: float = 0.0 def to_dict(self) -> Dict[str, Any]: - return { - "enabled": self.enabled, - "transport": self.transport, - "edit_interval": self.edit_interval, - "buffer_threshold": self.buffer_threshold, - "cursor": self.cursor, - "fresh_final_after_seconds": self.fresh_final_after_seconds, - } + return asdict(self) @classmethod def from_dict(cls, data: Dict[str, Any]) -> "StreamingConfig": if not isinstance(data, dict) or not data: return cls() - # ``mode`` is an ergonomic alias for the transport that ALSO implies - # ``enabled``. A config like ``streaming: {mode: auto}`` reads as - # "turn streaming on, transport=auto" — matching the natural intent - # of someone enabling streaming without also spelling out - # ``enabled: true``. Without this, ``mode`` was silently ignored and - # streaming stayed disabled (``enabled`` defaults to False), which is - # a surprising footgun: the whole reply buffers and sends at once. - # ``mode: off`` disables streaming; an explicit ``enabled`` key always - # wins so callers can force either state. - # - # ``transport`` alone does NOT imply ``enabled``: ``streaming.enabled`` - # is the documented master switch (see website/docs/user-guide/ - # configuration.md), so a bare ``transport`` only selects HOW to stream - # once streaming is on. Only the ``mode`` alias flips ``enabled``. + # ``mode`` is a transport alias that ALSO implies ``enabled`` (``mode: off`` disables; + # explicit ``enabled`` wins). A bare ``transport`` does NOT imply enabled: + # ``streaming.enabled`` is the documented master switch. raw_transport = data.get("transport") raw_mode = data.get("mode") - # Normalize both through the same helper so YAML's bare ``off``/``on`` - # (parsed as bool False/True) become canonical tokens rather than - # ``"false"``/``"true"``. - picked = raw_transport if raw_transport is not None else raw_mode - transport = _normalize_transport_token(picked) + transport = _normalize_transport_token(raw_transport if raw_transport is not None else raw_mode) if "enabled" in data: enabled = _coerce_bool(data.get("enabled"), False) elif raw_mode is not None: - # The ``mode`` alias (and only ``mode``) infers enabled: - # ``off`` disables, anything else enables. enabled = _normalize_transport_token(raw_mode) != "off" else: enabled = False @@ -763,33 +520,16 @@ class StreamingConfig: return cls( enabled=enabled, transport=transport, - edit_interval=_coerce_float( - data.get("edit_interval"), DEFAULT_STREAMING_EDIT_INTERVAL, - ), - buffer_threshold=_coerce_int( - data.get("buffer_threshold"), DEFAULT_STREAMING_BUFFER_THRESHOLD, - ), + edit_interval=_coerce_float(data.get("edit_interval"), DEFAULT_STREAMING_EDIT_INTERVAL), + buffer_threshold=_coerce_int(data.get("buffer_threshold"), DEFAULT_STREAMING_BUFFER_THRESHOLD), cursor=data.get("cursor", DEFAULT_STREAMING_CURSOR), - fresh_final_after_seconds=_coerce_float( - data.get("fresh_final_after_seconds"), 0.0 - ), + fresh_final_after_seconds=_coerce_float(data.get("fresh_final_after_seconds"), 0.0), ) -# ----------------------------------------------------------------------------- -# Built-in platform connection checkers -# ----------------------------------------------------------------------------- -# Each callable receives a ``PlatformConfig`` and returns ``True`` when the -# platform is sufficiently configured to be considered "connected". Platforms -# that rely on the generic ``token or api_key`` check (Telegram, Discord, -# Slack, Matrix, Mattermost, HomeAssistant) do not need an entry here. def _has_usable_api_server_key(key: object) -> bool: - """True when API_SERVER_KEY is present and strong enough to be usable. - - Mirrors the startup guard in ``gateway/platforms/api_server.py`` - (``has_usable_secret`` with ``min_length=16``) so the platform is only - enrolled at load time when the adapter would actually agree to start. - """ + """True when API_SERVER_KEY is strong enough for the adapter to start (mirrors the + ``has_usable_secret(min_length=16)`` guard in ``gateway/platforms/api_server.py``).""" if not key: return False try: @@ -799,169 +539,101 @@ def _has_usable_api_server_key(key: object) -> bool: return has_usable_secret(key, min_length=16) +def _needs_extra(*keys: str) -> Callable[[PlatformConfig], bool]: + return lambda cfg: all(cfg.extra.get(k) for k in keys) + + +# Built-in "sufficiently configured?" checks; platforms covered by the generic +# ``token or api_key`` check (Telegram, Discord, Slack, Matrix, ...) need no entry. _PLATFORM_CONNECTED_CHECKERS: dict[Platform, Callable[[PlatformConfig], bool]] = { - Platform.WEIXIN: lambda cfg: bool( - cfg.extra.get("account_id") and (cfg.token or cfg.extra.get("token")) - ), - Platform.WHATSAPP_CLOUD: lambda cfg: bool( - cfg.extra.get("phone_number_id") and cfg.extra.get("access_token") - ), - Platform.SIGNAL: lambda cfg: bool(cfg.extra.get("http_url")), - Platform.API_SERVER: lambda cfg: _has_usable_api_server_key( - cfg.extra.get("key") if cfg else None - ), + Platform.WEIXIN: lambda cfg: bool(cfg.extra.get("account_id") and (cfg.token or cfg.extra.get("token"))), + Platform.WHATSAPP_CLOUD: _needs_extra("phone_number_id", "access_token"), + Platform.SIGNAL: _needs_extra("http_url"), + Platform.API_SERVER: lambda cfg: _has_usable_api_server_key(cfg.extra.get("key") if cfg else None), Platform.WEBHOOK: lambda cfg: True, - Platform.MSGRAPH_WEBHOOK: lambda cfg: bool( - str(cfg.extra.get("client_state") or "").strip() - ), - Platform.BLUEBUBBLES: lambda cfg: bool( - cfg.extra.get("server_url") and cfg.extra.get("password") - ), - Platform.QQBOT: lambda cfg: bool( - cfg.extra.get("app_id") and cfg.extra.get("client_secret") - ), - Platform.YUANBAO: lambda cfg: bool( - cfg.extra.get("app_id") and cfg.extra.get("app_secret") - ), - # Relay dials OUT to a connector; it is "connected" once an endpoint URL is - # configured (extra["relay_url"] or extra["url"]). The capability descriptor - # is negotiated at handshake time, so the URL is the only config-level - # signal in the experimental phase. EXPERIMENTAL — may change. - Platform.RELAY: lambda cfg: bool( - cfg.extra.get("relay_url") or cfg.extra.get("url") - ), + Platform.MSGRAPH_WEBHOOK: lambda cfg: bool(str(cfg.extra.get("client_state") or "").strip()), + Platform.BLUEBUBBLES: _needs_extra("server_url", "password"), + Platform.QQBOT: _needs_extra("app_id", "client_secret"), + Platform.YUANBAO: _needs_extra("app_id", "app_secret"), + # Relay dials OUT: "connected" once an endpoint URL is configured. EXPERIMENTAL. + Platform.RELAY: lambda cfg: bool(cfg.extra.get("relay_url") or cfg.extra.get("url")), +} + + +# Top-level bool-ish keys read verbatim (no nested ``gateway.`` fallback) with their defaults. +_TOPLEVEL_BOOL_DEFAULTS = { + "write_sessions_json": True, "always_log_local": True, "filter_silence_narration": True, + "group_sessions_per_user": True, "thread_sessions_per_user": False, } @dataclass class GatewayConfig: - """ - Main gateway configuration. - - Manages all platform connections, session policies, and delivery settings. - """ - # Platform configurations + """Main gateway configuration: platform connections, session policies, delivery settings.""" platforms: Dict[Platform, PlatformConfig] = field(default_factory=dict) - - # Session reset policies by type default_reset_policy: SessionResetPolicy = field(default_factory=SessionResetPolicy) reset_by_type: Dict[str, SessionResetPolicy] = field(default_factory=dict) reset_by_platform: Dict[Platform, SessionResetPolicy] = field(default_factory=dict) - - # Reset trigger commands reset_triggers: List[str] = field(default_factory=lambda: ["/new", "/reset"]) - - # User-defined quick commands (slash commands that bypass the agent loop) + # Slash commands that bypass the agent loop. quick_commands: Dict[str, Any] = field(default_factory=dict) - - # Storage paths sessions_dir: Path = field(default_factory=lambda: get_hermes_home() / "sessions") - - # Whether to keep writing the legacy sessions.json mirror of the gateway - # routing index. The primary copy lives in state.db (gateway_routing - # table, #9006). Default True for backward compatibility with external - # tooling and downgrade safety; set gateway.write_sessions_json: false in - # config.yaml to stop producing the file. + # Keep the legacy sessions.json mirror of the routing index (primary: state.db) + # for external tooling and downgrade safety. write_sessions_json: bool = True - - # Delivery settings always_log_local: bool = True # Always save cron outputs to local files - # Drop outbound "silence narration" messages (e.g. *(silent)*, 🔇, a bare - # ".") pre-send. These are model hallucinations emitted when a persona has - # nothing actionable to say; in bot-to-bot channels they mirror back and - # forth, burning tokens and crashing models. Substrate-level guard that - # survives SOUL.md/prompt drift across providers. Opt out with False for - # raw passthrough. + # Drop outbound "silence narration" (*(silent)*, 🔇, a bare ".") that ping-pongs in + # bot-to-bot channels; a substrate guard that survives prompt drift. filter_silence_narration: bool = True - - # STT settings - stt_enabled: bool = True # Whether to auto-transcribe inbound voice messages - stt_echo_transcripts: bool = True # Whether to echo raw STT transcripts back to the user - - # Session isolation in shared chats - group_sessions_per_user: bool = True # Isolate group/channel sessions per participant when user IDs are available - thread_sessions_per_user: bool = False # When False (default), threads are shared across all participants - max_concurrent_sessions: Optional[int] = None # Positive int caps simultaneous active chat sessions - - # Multi-profile multiplexing (opt-in; default off preserves one-gateway-per-profile). - # When True, the default profile's gateway serves inbound messages for every - # profile on the host: profiles are stamped into session keys and (in later - # phases) per-profile adapters/credentials are resolved. When False, the - # gateway behaves exactly as before — single HERMES_HOME, no profile stamping. + stt_enabled: bool = True # Auto-transcribe inbound voice messages + stt_echo_transcripts: bool = True # Echo raw STT transcripts back to the user + group_sessions_per_user: bool = True # Isolate group sessions per participant when user IDs exist + thread_sessions_per_user: bool = False # False = threads shared across participants + max_concurrent_sessions: Optional[int] = None # Positive int caps simultaneous active sessions + # Opt-in: the default profile's gateway serves every profile on the host + # (profiles stamped into session keys, per-profile adapters/credentials). multiplex_profiles: bool = False - # Optional named-profile allowlist for multiplex mode. None preserves the - # historical serve-all behavior; [] serves only the default profile. + # None = historical serve-all; [] = default profile only. multiplex_profile_allowlist: Optional[List[str]] = None - - # Public HTTPS endpoint another gateway may use for scoped RoomLink calls. - # Disabled by default: setting an API key alone must never expose or - # advertise a route. HERMES_ROOM_LINK_URL remains the operator override. + # Public HTTPS endpoint for scoped RoomLink calls; an API key alone must never + # advertise a route. HERMES_ROOM_LINK_URL overrides. room_link_url: Optional[str] = None - - # Opt-in systemd event-loop watchdog. Zero preserves Type=simple and - # disables sd_notify at runtime. + # Opt-in systemd event-loop watchdog; zero keeps Type=simple and disables sd_notify. systemd_watchdog_seconds: int = 0 - - # In-process event-loop liveness watchdog (#69089). A daemon OS thread - # probes the gateway loop with call_soon_threadsafe; after consecutive - # missed probes it dumps all-thread stacks and hard-exits with the - # service-restart code so the supervisor can revive the process. On by - # default; set gateway.loop_watchdog: false in config.yaml to disable. - # - # Tuning knobs (all seconds unless noted) make the watchdog tolerate - # *transient, self-recovering* event-loop stalls — e.g. Telegram/Discord - # reconnect doing synchronous socket I/O during a network blip — so a - # short block does not force exit code 75 and trigger a restart churn - # that stalls cron dispatch (recurring fleet incidents on 2026-08-17, - # kanban t_0f76430f/t_70483f23). A genuine wedge (event loop frozen for - # the full tolerance window) still escalates to a supervised restart. + # In-process loop liveness watchdog: after consecutive missed probes it dumps + # all-thread stacks and hard-exits with the service-restart code. The knobs + # tolerate transient self-recovering stalls (adapter reconnect doing sync socket + # I/O) so a short block does not cause restart churn; a genuine wedge still escalates. loop_watchdog: bool = True - # Seconds the watchdog waits between liveness probes. loop_watchdog_probe_interval_s: float = DEFAULT_LOOP_WATCHDOG_INTERVAL_S - # Seconds a single probe may go unprocessed before it counts as a miss. loop_watchdog_probe_timeout_s: float = DEFAULT_LOOP_WATCHDOG_TIMEOUT_S - # Consecutive missed probes allowed before the watchdog hard-exits. - # Default stays at 3 (~90-120s of sustained loop block): the transient - # false-positive class (the watchdog's own on-loop heartbeat fsync) - # is fixed at the root by the off-loop write + two-witness probe, so - # raising this fleet-wide would only delay genuine-wedge recovery. + # ~90-120s sustained block. The heartbeat-fsync false positive is fixed at the root + # (off-loop write + two-witness probe), so raising this would only delay recovery. loop_watchdog_max_strikes: int = DEFAULT_LOOP_WATCHDOG_MAX_STRIKES - - # Unauthorized DM policy unauthorized_dm_behavior: str = "pair" # "pair" or "ignore" - - # Streaming configuration streaming: StreamingConfig = field(default_factory=StreamingConfig) - - # Session store pruning: drop SessionEntry records older than this many - # days from the in-memory dict and sessions.json. Keeps the store from - # growing unbounded in gateways serving many chats/threads/users over - # months. Pruning is invisible to users — if they resume, they get a - # fresh session exactly as if the reset policy had fired. 0 = disabled. + # Prune SessionEntry records older than this (a resumed chat gets a fresh session). 0 = off. session_store_max_age_days: int = 90 - - # Profile-based routing: route specific guilds/channels/threads to - # different profiles. See gateway/profile_routing.py. Each entry is a - # dict with: name, platform, profile, and optional guild_id/chat_id/thread_id. + # Route guilds/channels/threads to profiles (gateway/profile_routing.py). profile_routes: list = field(default_factory=list) + # Scalar fields serialized verbatim by ``to_dict`` (in output order). + _SCALAR_DICT_FIELDS = ( + "write_sessions_json", "always_log_local", "filter_silence_narration", "stt_enabled", + "stt_echo_transcripts", "group_sessions_per_user", "thread_sessions_per_user", + "max_concurrent_sessions", "multiplex_profiles", "multiplex_profile_allowlist", + "room_link_url", "systemd_watchdog_seconds", "loop_watchdog", + "loop_watchdog_probe_interval_s", "loop_watchdog_probe_timeout_s", + "loop_watchdog_max_strikes", "unauthorized_dm_behavior", + ) + def __post_init__(self) -> None: - self.multiplex_profile_allowlist = _normalize_multiplex_profile_allowlist( - self.multiplex_profile_allowlist - ) - self.systemd_watchdog_seconds = coerce_systemd_watchdog_seconds( - self.systemd_watchdog_seconds - ) + self.multiplex_profile_allowlist = _normalize_multiplex_profile_allowlist(self.multiplex_profile_allowlist) + self.systemd_watchdog_seconds = coerce_systemd_watchdog_seconds(self.systemd_watchdog_seconds) def get_connected_platforms(self) -> List[Platform]: - """Return list of platforms that are enabled and configured. - - Sorted by platform value so the rendered "Connected Platforms" list - (and the home-channel blocks derived from it) is byte-stable across - gateway restarts and mid-process platform registration — dict - insertion order is not a stable contract and a reorder busts the - prompt cache without any semantic change. - """ + """Enabled + configured platforms, sorted by value so the rendered "Connected + Platforms" prompt block is byte-stable (a reorder busts the prompt cache).""" connected = [ platform for platform, config in self.platforms.items() @@ -970,32 +642,21 @@ class GatewayConfig: return sorted(connected, key=lambda p: str(p.value)) def _is_platform_connected(self, platform: Platform, config: PlatformConfig) -> bool: - """Check whether a single platform is sufficiently configured.""" checker = _PLATFORM_CONNECTED_CHECKERS.get(platform) - # Weixin requires both a token and an account_id (checked first so - # the generic token branch doesn't let it through without account_id). + # Weixin needs token AND account_id, so it must bypass the generic token branch. if platform == Platform.WEIXIN: return checker(config) - - # Generic token/api_key auth covers Telegram, Discord, Slack, etc. if config.token or config.api_key: return True - - # Platform-specific check if checker is not None: return checker(config) - # Plugin-registered platforms. Force plugin discovery first so this - # works even when GatewayConfig is constructed directly (e.g. in tests - # or callers that bypass load_gateway_config(), which is what triggers - # discovery in the normal path). discover_plugins() is idempotent. + # Plugin platforms; force (idempotent) discovery for directly-constructed configs. try: from gateway.platform_registry import platform_registry - try: + with contextlib.suppress(Exception): from hermes_cli.plugins import discover_plugins discover_plugins() - except Exception: - pass entry = platform_registry.get(platform.value) if entry: if entry.is_connected is not None: @@ -1005,74 +666,40 @@ class GatewayConfig: return True except Exception: pass # Registry not yet initialised during early import - return False - + def get_home_channel(self, platform: Platform) -> Optional[HomeChannel]: - """Get the home channel for a platform.""" config = self.platforms.get(platform) return config.home_channel if config else None - - def get_reset_policy( - self, - platform: Optional[Platform] = None, - session_type: Optional[str] = None - ) -> SessionResetPolicy: - """ - Get the appropriate reset policy for a session. - - Priority: platform override > type override > default - """ - # Platform-specific override takes precedence + + def get_reset_policy(self, platform: Optional[Platform] = None, session_type: Optional[str] = None) -> SessionResetPolicy: + """Priority: platform override > type override > default.""" if platform and platform in self.reset_by_platform: return self.reset_by_platform[platform] - - # Type-specific override (dm, group, thread) if session_type and session_type in self.reset_by_type: return self.reset_by_type[session_type] - return self.default_reset_policy - + def to_dict(self) -> Dict[str, Any]: - return { - "platforms": { - p.value: c.to_dict() for p, c in self.platforms.items() - }, + result: Dict[str, Any] = { + "platforms": {p.value: c.to_dict() for p, c in self.platforms.items()}, "default_reset_policy": self.default_reset_policy.to_dict(), - "reset_by_type": { - k: v.to_dict() for k, v in self.reset_by_type.items() - }, - "reset_by_platform": { - p.value: v.to_dict() for p, v in self.reset_by_platform.items() - }, + "reset_by_type": {k: v.to_dict() for k, v in self.reset_by_type.items()}, + "reset_by_platform": {p.value: v.to_dict() for p, v in self.reset_by_platform.items()}, "reset_triggers": self.reset_triggers, "quick_commands": self.quick_commands, "sessions_dir": str(self.sessions_dir), - "write_sessions_json": self.write_sessions_json, - "always_log_local": self.always_log_local, - "filter_silence_narration": self.filter_silence_narration, - "stt_enabled": self.stt_enabled, - "stt_echo_transcripts": self.stt_echo_transcripts, - "group_sessions_per_user": self.group_sessions_per_user, - "thread_sessions_per_user": self.thread_sessions_per_user, - "max_concurrent_sessions": self.max_concurrent_sessions, - "multiplex_profiles": self.multiplex_profiles, - "multiplex_profile_allowlist": self.multiplex_profile_allowlist, - "room_link_url": self.room_link_url, - "systemd_watchdog_seconds": self.systemd_watchdog_seconds, - "loop_watchdog": self.loop_watchdog, - "loop_watchdog_probe_interval_s": self.loop_watchdog_probe_interval_s, - "loop_watchdog_probe_timeout_s": self.loop_watchdog_probe_timeout_s, - "loop_watchdog_max_strikes": self.loop_watchdog_max_strikes, - "unauthorized_dm_behavior": self.unauthorized_dm_behavior, - "streaming": self.streaming.to_dict(), - "session_store_max_age_days": self.session_store_max_age_days, - "profile_routes": [ - asdict(r) if is_dataclass(r) and not isinstance(r, type) else r - for r in self.profile_routes - ], } - + for name in self._SCALAR_DICT_FIELDS: + result[name] = getattr(self, name) + result["streaming"] = self.streaming.to_dict() + result["session_store_max_age_days"] = self.session_store_max_age_days + result["profile_routes"] = [ + asdict(r) if is_dataclass(r) and not isinstance(r, type) else r + for r in self.profile_routes + ] + return result + @classmethod def from_dict(cls, data: Dict[str, Any]) -> "GatewayConfig": data = _coerce_dict(data) @@ -1082,45 +709,32 @@ class GatewayConfig: """Top-level key wins by presence; else the nested ``gateway.`` form.""" return data[key] if key in data else nested_gateway.get(key) - platforms = {} - for platform_name, platform_data in _coerce_dict(data.get("platforms", {})).items(): - if not isinstance(platform_data, dict): - continue - try: - platforms[Platform(platform_name)] = PlatformConfig.from_dict(platform_data) - except ValueError: - pass # Skip unknown platforms - - reset_by_platform = {} - for platform_name, policy_data in _coerce_dict(data.get("reset_by_platform", {})).items(): - try: - reset_by_platform[Platform(platform_name)] = SessionResetPolicy.from_dict(policy_data) - except ValueError: - pass - - stt = _coerce_dict(data.get("stt")) - stt_enabled = data.get("stt_enabled") - if stt_enabled is None: - stt_enabled = stt.get("enabled") - stt_echo_transcripts = data.get("stt_echo_transcripts") - if stt_echo_transcripts is None: - stt_echo_transcripts = stt.get("echo_transcripts") - - room_link_url = data.get("room_link_url") - if not isinstance(room_link_url, str): - room_link_url = None - - # Key prefix for the warning: "gateway." when the nested form was the one consulted. def key_label(key: str) -> str: + """Warning key prefix: "gateway." when the nested form was the one consulted.""" return key if key in data else f"gateway.{key}" - # Watchdog knobs: out-of-range / non-finite values fall back to the shipped defaults. - probe_interval = _coerce_float(pick("loop_watchdog_probe_interval_s"), DEFAULT_LOOP_WATCHDOG_INTERVAL_S) - if not math.isfinite(probe_interval) or not 1.0 <= probe_interval <= 3600.0: - probe_interval = DEFAULT_LOOP_WATCHDOG_INTERVAL_S - probe_timeout = _coerce_float(pick("loop_watchdog_probe_timeout_s"), DEFAULT_LOOP_WATCHDOG_TIMEOUT_S) - if not math.isfinite(probe_timeout) or not 1.0 <= probe_timeout <= 600.0: - probe_timeout = DEFAULT_LOOP_WATCHDOG_TIMEOUT_S + def by_platform(key: str, parse, *, dicts_only: bool = False) -> dict: + """``{Platform(name): parse(block)}`` for a platform-keyed mapping; unknown platforms skipped.""" + out = {} + for platform_name, block in _coerce_dict(data.get(key, {})).items(): + if dicts_only and not isinstance(block, dict): + continue + try: + out[Platform(platform_name)] = parse(block) + except ValueError: + pass + return out + + def stt_setting(flat_key: str, nested_key: str) -> Any: + value = data.get(flat_key) + return _coerce_dict(data.get("stt")).get(nested_key) if value is None else value + + def bounded_float(key: str, default: float, lo: float, hi: float) -> float: + # Out-of-range / non-finite watchdog knobs fall back to the shipped defaults. + value = _coerce_float(pick(key), default) + return value if math.isfinite(value) and lo <= value <= hi else default + + room_link_url = data.get("room_link_url") max_strikes = _coerce_int(pick("loop_watchdog_max_strikes"), DEFAULT_LOOP_WATCHDOG_MAX_STRIKES) if not 1 <= max_strikes <= 1000: max_strikes = DEFAULT_LOOP_WATCHDOG_MAX_STRIKES @@ -1129,14 +743,9 @@ class GatewayConfig: pick("systemd_watchdog_seconds"), key_label("systemd_watchdog_seconds") ) - # Multiplexing is a genuine 3-tier chain: env > config.yaml > default False. The - # GATEWAY_MULTIPLEX_PROFILES operator override wins when set to a recognized value - # (hosted deployments stamp it on the container so the single multiplexed gateway the - # connector depends on is forced on at every boot regardless of the image's config.yaml); - # a blank or unrecognized env value falls through to config — a provisioned-but- - # unpopulated Fly secret must not shadow a config.yaml opt-in. Config side: the - # top-level VALUE wins when not None, else ``gateway.multiplex_profiles`` (written by - # ``hermes config set gateway.multiplex_profiles true``). + # env > config.yaml > False: a recognized GATEWAY_MULTIPLEX_PROFILES wins (hosted + # deployments stamp it on the container); blank/unrecognized falls through to the + # top-level VALUE when not None, else ``gateway.multiplex_profiles``. multiplex_profiles = data.get("multiplex_profiles") if multiplex_profiles is None: multiplex_profiles = nested_gateway.get("multiplex_profiles") @@ -1153,12 +762,10 @@ class GatewayConfig: except (TypeError, ValueError): session_store_max_age_days = 90 - # Parse profile routes (validated by gateway.profile_routing) from gateway.profile_routing import parse_profile_routes - profile_routes = parse_profile_routes(data.get("profile_routes") or []) return cls( - platforms=platforms, + platforms=by_platform("platforms", PlatformConfig.from_dict, dicts_only=True), default_reset_policy=SessionResetPolicy.from_dict(data["default_reset_policy"]) if "default_reset_policy" in data else SessionResetPolicy(), @@ -1166,24 +773,20 @@ class GatewayConfig: type_name: SessionResetPolicy.from_dict(policy_data) for type_name, policy_data in _coerce_dict(data.get("reset_by_type", {})).items() }, - reset_by_platform=reset_by_platform, + reset_by_platform=by_platform("reset_by_platform", SessionResetPolicy.from_dict), reset_triggers=data.get("reset_triggers", ["/new", "/reset"]), quick_commands=_coerce_dict(data.get("quick_commands", {})), sessions_dir=Path(data["sessions_dir"]) if "sessions_dir" in data else get_hermes_home() / "sessions", - write_sessions_json=_coerce_bool(data.get("write_sessions_json"), True), - always_log_local=_coerce_bool(data.get("always_log_local"), True), - filter_silence_narration=_coerce_bool(data.get("filter_silence_narration"), True), - stt_enabled=_coerce_bool(stt_enabled, True), - stt_echo_transcripts=_coerce_bool(stt_echo_transcripts, True), - group_sessions_per_user=_coerce_bool(data.get("group_sessions_per_user"), True), - thread_sessions_per_user=_coerce_bool(data.get("thread_sessions_per_user"), False), + **{name: _coerce_bool(data.get(name), default) for name, default in _TOPLEVEL_BOOL_DEFAULTS.items()}, + stt_enabled=_coerce_bool(stt_setting("stt_enabled", "enabled"), True), + stt_echo_transcripts=_coerce_bool(stt_setting("stt_echo_transcripts", "echo_transcripts"), True), multiplex_profiles=_coerce_bool(multiplex_profiles, False), multiplex_profile_allowlist=pick("multiplex_profile_allowlist"), - room_link_url=room_link_url, + room_link_url=room_link_url if isinstance(room_link_url, str) else None, systemd_watchdog_seconds=systemd_watchdog_seconds, loop_watchdog=_coerce_bool(pick("loop_watchdog"), True), - loop_watchdog_probe_interval_s=probe_interval, - loop_watchdog_probe_timeout_s=probe_timeout, + loop_watchdog_probe_interval_s=bounded_float("loop_watchdog_probe_interval_s", DEFAULT_LOOP_WATCHDOG_INTERVAL_S, 1.0, 3600.0), + loop_watchdog_probe_timeout_s=bounded_float("loop_watchdog_probe_timeout_s", DEFAULT_LOOP_WATCHDOG_TIMEOUT_S, 1.0, 600.0), loop_watchdog_max_strikes=max_strikes, max_concurrent_sessions=max_concurrent_sessions, unauthorized_dm_behavior=_normalize_choice( @@ -1191,16 +794,13 @@ class GatewayConfig: ), streaming=StreamingConfig.from_dict(data.get("streaming", {})), session_store_max_age_days=session_store_max_age_days, - profile_routes=profile_routes, + profile_routes=parse_profile_routes(data.get("profile_routes") or []), ) def get_unauthorized_dm_behavior(self, platform: Optional[Platform] = None) -> str: - """Return the effective unauthorized-DM behavior for a platform. - - Email is inbox-shaped, not chat-shaped, so it defaults to ``"ignore"`` - unless ``platforms.email.unauthorized_dm_behavior`` explicitly opts - into pairing. A global default does not opt email into pairing. - """ + """Effective unauthorized-DM behavior. Email is inbox-shaped so it defaults to + ``"ignore"`` unless its own ``unauthorized_dm_behavior`` opts in (a global + default does not).""" if platform: platform_cfg = self.platforms.get(platform) if platform_cfg and "unauthorized_dm_behavior" in platform_cfg.extra: @@ -1214,7 +814,7 @@ class GatewayConfig: return self.unauthorized_dm_behavior def get_notice_delivery(self, platform: Optional[Platform] = None) -> str: - """Return the effective notice-delivery mode for a platform.""" + """Effective notice-delivery mode ("public"/"private") for a platform.""" if platform: platform_cfg = self.platforms.get(platform) if platform_cfg and "notice_delivery" in platform_cfg.extra: @@ -1225,15 +825,7 @@ class GatewayConfig: def load_gateway_config() -> GatewayConfig: - """ - Load gateway configuration from multiple sources. - - Priority (highest to lowest): - 1. Environment variables - 2. ~/.hermes/config.yaml (primary user-facing config) - 3. ~/.hermes/gateway.json (legacy — provides defaults under config.yaml) - 4. Built-in defaults - """ + """Load gateway configuration. Priority: env > ~/.hermes/config.yaml > legacy gateway.json > defaults.""" from gateway import config_loader _home = get_hermes_home() @@ -1244,8 +836,7 @@ def load_gateway_config() -> GatewayConfig: logger.warning( "Failed to process config.yaml — falling back to .env / gateway.json values. " "Check %s for syntax errors. Error: %s", - _home / "config.yaml", - e, + _home / "config.yaml", e, ) config = GatewayConfig.from_dict(gw_data) @@ -1255,54 +846,36 @@ def load_gateway_config() -> GatewayConfig: def _validate_gateway_config(config: "GatewayConfig") -> None: - """Validate and sanitize a loaded GatewayConfig in place. - - Called by ``load_gateway_config()`` after all config sources are merged. - Extracted as a separate function for testability. - """ + """Validate and sanitize a loaded GatewayConfig in place (after all sources are merged).""" policy = config.default_reset_policy if not (0 <= policy.at_hour <= 23): - logger.warning( - "Invalid at_hour=%s (must be 0-23). Using default 4.", policy.at_hour - ) + logger.warning("Invalid at_hour=%s (must be 0-23). Using default 4.", policy.at_hour) policy.at_hour = 4 if policy.idle_minutes is None or policy.idle_minutes <= 0: - logger.warning( - "Invalid idle_minutes=%s (must be positive). Using default 1440.", - policy.idle_minutes, - ) + logger.warning("Invalid idle_minutes=%s (must be positive). Using default 1440.", policy.idle_minutes) policy.idle_minutes = 1440 - # Warn about empty bot tokens — platforms that loaded an empty string - # won't connect and the cause can be confusing without a log line. - for platform, pconfig in config.platforms.items(): - if not pconfig.enabled: - continue - env_name = PLATFORM_TOKEN_ENV_NAMES.get(platform) - if env_name and pconfig.token is not None and not pconfig.token.strip(): - logger.warning( - "%s is enabled but %s is empty. " - "The adapter will likely fail to connect.", - platform.value, env_name, - ) - - # Reject known-weak placeholder tokens. - # Ported from openclaw/openclaw#64586: users who copy .env.example - # without changing placeholder values get a clear startup error instead - # of a confusing "auth failed" from the platform API. try: from hermes_cli.auth import has_usable_secret except ImportError: - return + has_usable_secret = None - for platform, pconfig in config.platforms.items(): - env_name = PLATFORM_TOKEN_ENV_NAMES.get(platform) + token_platforms = [ + (p, c, PLATFORM_TOKEN_ENV_NAMES[p]) for p, c in config.platforms.items() + if c.enabled and p in PLATFORM_TOKEN_ENV_NAMES and c.token is not None + ] + # An empty token won't connect; say so. + for platform, pconfig, env_name in token_platforms: + if not pconfig.token.strip(): + logger.warning("%s is enabled but %s is empty. The adapter will likely fail to connect.", platform.value, env_name) + if has_usable_secret is None: + return + # Reject placeholder tokens (copied .env.example) with a clear startup error. + for platform, pconfig, env_name in token_platforms: token = pconfig.token - if not (pconfig.enabled and env_name and token and token.strip()): - continue - if not has_usable_secret(token, min_length=4): + if token.strip() and not has_usable_secret(token, min_length=4): logger.error( "%s is enabled but %s is set to a placeholder value ('%s'). " "Set a real bot token before starting the gateway. " diff --git a/gateway/config_env.py b/gateway/config_env.py index fc56881975..f448a0d7e0 100644 --- a/gateway/config_env.py +++ b/gateway/config_env.py @@ -2,9 +2,9 @@ Runs after ``GatewayConfig.from_dict`` so env always wins over config.yaml / gateway.json. Most platforms follow one shape — "credentials present in env -⇒ enable the platform and copy the values into ``extra``" — and are declared -as ``_Cred`` rows in ``_ENV_STEPS`` (source order = application order). -Platforms with unique gating live as small functions in the same table. +⇒ enable the platform and copy the values into ``extra``" — declared as +``_Cred`` rows in ``_ENV_STEPS`` (source order = application order). +Platforms with unique gating are small functions in the same table. """ import contextlib @@ -13,7 +13,7 @@ import logging import os from dataclasses import dataclass from functools import partial -from typing import Any, Callable, Dict, List, Optional +from typing import Any, Callable, Dict, Optional from gateway.config import ( GatewayConfig, @@ -30,15 +30,13 @@ logger = logging.getLogger("gateway.config") getenv = _getenv_str -# Platforms for which the "explicitly disabled in config.yaml, but credentials -# are present in the environment" WARNING has already been emitted in this -# process. Config reloads on every turn, so the notice is one-time per platform -# per process — loud once at startup, never a per-turn drumbeat. +# Platforms already warned about "explicitly disabled in config.yaml, but +# credentials present in env". Config reloads every turn, so the notice is +# one-time per platform per process. _EXPLICIT_DISABLE_WARNED: set = set() - # Env var(s) whose presence drives each platform's env-enable branch, named in -# the explicit-disable WARNING below. +# the explicit-disable WARNING. _ENV_ENABLE_CREDENTIALS: dict = { Platform.TELEGRAM: ("TELEGRAM_BOT_TOKEN",), Platform.DISCORD: ("DISCORD_BOT_TOKEN",), @@ -63,13 +61,8 @@ _ENV_ENABLE_CREDENTIALS: dict = { def _warn_explicit_disable_beats_env(platform: Platform) -> None: - """One-time WARNING: ``platforms..enabled: false`` wins over env creds. - - Until #48820 credential presence force-enabled twelve platforms regardless - of an explicit ``enabled: false``; users relying on "creds in .env = - platform on" need to be told why it went dark. Names the platform, the - winning config key, and the env var(s) that used to override it. - """ + """One-time WARNING: ``platforms..enabled: false`` wins over env creds (which + used to force-enable), so users relying on "creds in .env = on" learn why it went dark.""" if platform in _EXPLICIT_DISABLE_WARNED: return _EXPLICIT_DISABLE_WARNED.add(platform) @@ -88,12 +81,12 @@ def _warn_explicit_disable_beats_env(platform: Platform) -> None: # --- small value parsers ----------------------------------------------------- -def _csv_list(value: str) -> List[str]: +def _csv_list(value: str) -> list: return [part.strip() for part in value.split(",") if part.strip()] def _int_or(default: int) -> Callable[[str], int]: - """``int(raw.strip(), 10)`` with *default* on malformed/blank input (``_getenv_int`` semantics).""" + """``int(raw.strip(), 10)`` with *default* on malformed/blank input.""" def parse(raw: str) -> int: try: return int(str(raw).strip(), 10) @@ -106,6 +99,10 @@ def _strip_slash(value: str) -> str: return value.rstrip("/") +def _strip_lower(value: str) -> str: + return value.strip().lower() + + def _truthy_token(value: str) -> bool: return value.lower() in {"true", "1", "yes", "on"} @@ -121,11 +118,7 @@ def _env_first(envs) -> str: """First truthy value among *envs* (a single name or a tuple of alternatives).""" if isinstance(envs, str): return getenv(envs) - for env in envs: - value = getenv(env) - if value: - return value - return "" + return next((v for v in map(getenv, envs) if v), "") # --- reusable steps ----------------------------------------------------------- @@ -136,8 +129,8 @@ _INT = object() # spec marker: ``int(value)``, silently skipped when malformed def _env_extras(extra: Dict[str, Any], spec, *, strip: bool = False) -> None: """``extra[key] = fn(value)`` for each ``(key, env[, fn])`` whose env value is truthy. - With ``strip=True`` the value is stripped BEFORE the truthiness check. ``fn=_INT`` parses an - int and silently ignores a non-integer value. + ``strip=True`` strips BEFORE the truthiness check. ``fn=_INT`` parses an int + and silently ignores a non-integer value. """ for key, env, *fn in spec: value = getenv(env) @@ -159,10 +152,8 @@ def _env_home_channel(config: GatewayConfig, platform: Platform, env_base: str, chat_id = chat_id.strip() if chat_id and platform in config.platforms: config.platforms[platform].home_channel = HomeChannel( - platform=platform, - chat_id=chat_id, - name=getenv(f"{env_base}_NAME", "Home"), - thread_id=getenv(f"{env_base}_THREAD_ID") or None, + platform=platform, chat_id=chat_id, + name=getenv(f"{env_base}_NAME", "Home"), thread_id=getenv(f"{env_base}_THREAD_ID") or None, ) @@ -173,18 +164,14 @@ def _env_reply_mode(config: GatewayConfig, platform: Platform, env: str) -> None def _enable_from_env(config: GatewayConfig, platform: Platform) -> PlatformConfig: - """Enable *platform* because its env credentials are present — unless config.yaml explicitly disabled it. - - READS (does not pop) the ``_enabled_explicit`` marker: the registry-driven plugin-enable pass later - also needs it. The marker is scrubbed for all platforms at the end of ``_apply_env_overrides``. - """ + """Enable *platform* on env credentials unless config.yaml explicitly disabled it. + READS (does not pop) ``_enabled_explicit``: the plugin-enable pass also needs it.""" if platform not in config.platforms: config.platforms[platform] = PlatformConfig(enabled=True) return config.platforms[platform] platform_config = config.platforms[platform] if not platform_config.enabled: if platform_config.extra.get("_enabled_explicit", False): - # Credentials are present (that is why we are here) but the user said no in config.yaml. _warn_explicit_disable_beats_env(platform) else: platform_config.enabled = True @@ -192,12 +179,11 @@ def _enable_from_env(config: GatewayConfig, platform: Platform) -> PlatformConfi def _enable_port_bound_from_env(config: GatewayConfig, platform: Platform) -> PlatformConfig: - """Enable a port-binding platform (api_server/webhook) unless config.yaml explicitly disabled it. + """Enable a port-binding platform unless config.yaml explicitly disabled it. - In multiplex mode a secondary profile pins ``platforms..enabled: false`` so it shares the - default profile's listener instead of binding its own port, yet still inherits the process env; - without this guard the env presence would force-enable the listener and trip MultiplexConfigError. - POPs the ``_enabled_explicit`` marker: these branches are terminal (no later registry pass). + A multiplex secondary profile pins ``enabled: false`` to share the default profile's + listener yet inherits the process env; without this guard env presence would + force-enable it and trip MultiplexConfigError. POPs the marker (terminal branch). """ platform_config = config.platforms.setdefault(platform, PlatformConfig()) explicit = platform_config.extra.pop("_enabled_explicit", False) @@ -210,12 +196,12 @@ def _enable_port_bound_from_env(config: GatewayConfig, platform: Platform) -> Pl class _Cred: """Credential-gated platform enable, applied as ``step(config)``. - ``creds``: env names that must ALL be truthy; an inner tuple lists alternatives (ANY). - ``token``: env whose truthy value becomes ``PlatformConfig.token``. - ``fixed``: ``(extra_key, env[, default[, fn]])`` always written once enabled (``env`` may be a - tuple of alternatives). ``optional`` / ``optional_stripped``: ``_env_extras`` specs, written only - when truthy. ``warn_missing``: ``(env, msg)`` logged BEFORE enabling when *env* is blank. ``then``: unique tail ``fn(config, platform_config)``. - ``home``: ``_env_home_channel`` env base applied only when the gate passed. + ``creds``: env names that must ALL be truthy (inner tuple = ANY of). ``token``: env + stored as ``PlatformConfig.token`` even when yaml disables the adapter (sending skills + use it). ``fixed``: ``(extra_key, env[, default[, fn]])`` always written once enabled. + ``optional*``: ``_env_extras`` specs. ``warn_missing``: ``(env, msg)`` logged BEFORE + enabling when blank. ``then``: tail ``fn(config, platform_config)``. ``home``: + ``_env_home_channel`` env base applied only when the gate passed. """ platform: Platform creds: tuple @@ -268,14 +254,12 @@ def _telegram_fallback_ips(config: GatewayConfig) -> None: def _whatsapp(config: GatewayConfig) -> None: - """WhatsApp (Baileys bridge) uses a flag, not credentials.""" + """WhatsApp (Baileys bridge) uses a flag, not credentials; an explicit false overrides YAML.""" raw = getenv("WHATSAPP_ENABLED") enabled = is_truthy_value(raw) - disabled_explicitly = raw.lower() in {"false", "0", "no"} - if Platform.WHATSAPP in config.platforms: - # YAML config exists — respect explicit disable; otherwise keep whatever the YAML set. - wa_cfg = config.platforms[Platform.WHATSAPP] - if disabled_explicitly: + wa_cfg = config.platforms.get(Platform.WHATSAPP) + if wa_cfg is not None: + if raw.lower() in {"false", "0", "no"}: wa_cfg.enabled = False elif enabled: wa_cfg.enabled = True @@ -283,34 +267,21 @@ def _whatsapp(config: GatewayConfig) -> None: config.platforms[Platform.WHATSAPP] = PlatformConfig(enabled=True) -def _slack(config: GatewayConfig) -> None: - slack_token = getenv("SLACK_BOT_TOKEN") - if slack_token: - if Platform.SLACK not in config.platforms: - config.platforms[Platform.SLACK] = PlatformConfig(enabled=True) - else: - slack_config = config.platforms[Platform.SLACK] - if not slack_config.enabled and not slack_config.extra.get("_enabled_explicit", False): - # Top-level Slack settings such as channel prompts must not turn an env-token setup - # into a disabled platform; only an explicit enabled: false should. - slack_config.enabled = True - elif not slack_config.enabled: - _warn_explicit_disable_beats_env(Platform.SLACK) - # Token is stored even when yaml disables the adapter so Slack-sending skills can use it. - config.platforms[Platform.SLACK].token = slack_token +def _slack_home(config: GatewayConfig) -> None: + """SLACK_HOME_CHANNEL creates a disabled Slack entry if needed and keeps + user_id/scope_id provenance when the chat_id is unchanged.""" slack_home = getenv("SLACK_HOME_CHANNEL") - if slack_home: - slack_config = config.platforms.setdefault(Platform.SLACK, PlatformConfig(enabled=False)) - existing_home = slack_config.home_channel - same_home = existing_home is not None and existing_home.chat_id == slack_home - slack_config.home_channel = HomeChannel( - platform=Platform.SLACK, - chat_id=slack_home, - name=getenv("SLACK_HOME_CHANNEL_NAME"), - thread_id=getenv("SLACK_HOME_CHANNEL_THREAD_ID") or None, - user_id=existing_home.user_id if same_home else None, - scope_id=existing_home.scope_id if same_home else None, - ) + if not slack_home: + return + slack_config = config.platforms.setdefault(Platform.SLACK, PlatformConfig(enabled=False)) + existing_home = slack_config.home_channel + same_home = existing_home is not None and existing_home.chat_id == slack_home + slack_config.home_channel = HomeChannel( + platform=Platform.SLACK, chat_id=slack_home, + name=getenv("SLACK_HOME_CHANNEL_NAME"), thread_id=getenv("SLACK_HOME_CHANNEL_THREAD_ID") or None, + user_id=existing_home.user_id if same_home else None, + scope_id=existing_home.scope_id if same_home else None, + ) def _matrix_e2ee(config: GatewayConfig, matrix_config: PlatformConfig) -> None: @@ -324,15 +295,12 @@ def _matrix_e2ee(config: GatewayConfig, matrix_config: PlatformConfig) -> None: _env_extras(matrix_config.extra, (("device_id", "MATRIX_DEVICE_ID"),)) -def _sms(config: GatewayConfig) -> None: - if getenv("TWILIO_ACCOUNT_SID"): - _enable_from_env(config, Platform.SMS).api_key = getenv("TWILIO_AUTH_TOKEN") +def _sms_api_key(config: GatewayConfig, sms_config: PlatformConfig) -> None: + sms_config.api_key = getenv("TWILIO_AUTH_TOKEN") def _api_server(config: GatewayConfig) -> None: - """Require a usable key: API_SERVER_ENABLED alone would load an unauthenticated platform whose - adapter refuses to start at connect() anyway, leaving the reconnect watcher spinning forever. - Same strength bar as the startup guard (has_usable_secret, min_length=16).""" + """Require a usable key: an unauthenticated adapter refuses to start and the reconnect watcher would spin.""" key = getenv("API_SERVER_KEY") if not _has_usable_api_server_key(key): return @@ -355,18 +323,11 @@ def _msgraph_webhook(config: GatewayConfig) -> None: client_state = getenv("MSGRAPH_WEBHOOK_CLIENT_STATE") resources = getenv("MSGRAPH_WEBHOOK_ACCEPTED_RESOURCES") allowed_cidrs = getenv("MSGRAPH_WEBHOOK_ALLOWED_SOURCE_CIDRS") - if not ( - enabled - or Platform.MSGRAPH_WEBHOOK in config.platforms - or getenv("MSGRAPH_WEBHOOK_PORT") - or client_state - or resources - or allowed_cidrs - ): + if not (enabled or Platform.MSGRAPH_WEBHOOK in config.platforms or getenv("MSGRAPH_WEBHOOK_PORT") + or client_state or resources or allowed_cidrs): return msgraph_cfg = config.platforms.setdefault(Platform.MSGRAPH_WEBHOOK, PlatformConfig()) - # Same explicit-disable guard as the webhook branch, but READ (don't pop) the marker: - # the relay-exclusive pass below still consults it; the end-of-function scrub removes it. + # Same guard as webhook, but READ (don't pop) the marker: the relay pass still consults it. if enabled and (not msgraph_cfg.extra.get("_enabled_explicit", False) or msgraph_cfg.enabled): msgraph_cfg.enabled = True _env_extras(msgraph_cfg.extra, (("port", "MSGRAPH_WEBHOOK_PORT", _INT),)) @@ -383,9 +344,8 @@ def _qq_home(config: GatewayConfig, qq_config: PlatformConfig) -> None: name_env = "QQBOT_HOME_CHANNEL_NAME" if not qq_home: # Back-compat: accept the pre-rename name and log a one-time warning. - legacy_home = getenv("QQ_HOME_CHANNEL").strip() - if legacy_home: - qq_home = legacy_home + qq_home = getenv("QQ_HOME_CHANNEL").strip() + if qq_home: name_env = "QQ_HOME_CHANNEL_NAME" logger.warning( "QQ_HOME_CHANNEL is deprecated; rename to QQBOT_HOME_CHANNEL " @@ -393,8 +353,7 @@ def _qq_home(config: GatewayConfig, qq_config: PlatformConfig) -> None: ) if qq_home: qq_config.home_channel = HomeChannel( - platform=Platform.QQBOT, - chat_id=qq_home, + platform=Platform.QQBOT, chat_id=qq_home, name=getenv("QQBOT_HOME_CHANNEL_NAME") or getenv(name_env, "Home"), thread_id=getenv("QQBOT_HOME_CHANNEL_THREAD_ID") or getenv("QQ_HOME_CHANNEL_THREAD_ID") or None, ) @@ -408,114 +367,106 @@ def _session_settings(config: GatewayConfig) -> None: setattr(config.default_reset_policy, attr, int(raw)) -def _enable_plugin_platforms_from_env(config: GatewayConfig) -> None: - """Registry-driven enable for plugin platforms (built-ins have explicit rows in ``_ENV_STEPS``). +def _plugin_probe_seed(entry) -> Optional[dict]: + """``env_enablement_fn()`` result as a non-empty dict, else None.""" + if entry.env_enablement_fn is None: + return None + try: + seed = entry.env_enablement_fn() + except Exception as e: + logger.debug("env_enablement_fn for %s raised: %s", entry.name, e) + return None + return seed if isinstance(seed, dict) and seed else None - A plugin platform is enabled when its credentials are configured (``is_connected``) and its deps - are present (passive ``check_fn``) or installable on demand (``ensure_deps_fn`` — run later by - ``create_adapter()``, never here: the active installer used to be wired as ``check_fn`` and this - sweep pip-installed SDKs on every ``load_gateway_config()`` call, boot-looping the desktop app). - ``is_connected`` MUST gate enablement: ``check_fn`` alone ("is the SDK importable?") would enable - platforms the user never configured and the gateway would retry-connect forever with no token. + +def _plugin_is_configured(entry, existing_extra: dict, seed: Optional[dict]) -> bool: + """``entry.is_connected`` on a transient ``enabled=True`` view seeded with env extras.""" + try: + # Seed extras so ``is_connected`` implementations reading ``config.extra`` see + # post-enable state; never mutate the real config for the probe. + for k, v in (seed or {}).items(): + if k != "home_channel": + existing_extra.setdefault(k, v) + configured = bool(entry.is_connected(PlatformConfig(enabled=True, extra=existing_extra))) + except Exception as exc: + logger.debug("is_connected for %s raised: %s — skipping enablement", entry.name, exc) + return False + if not configured: + logger.debug( + "Plugin platform '%s' available but not configured " + "(is_connected returned False) — skipping enable", + entry.name, + ) + return configured + + +def _enable_plugin_platform(config: GatewayConfig, entry) -> None: + try: + platform = Platform(entry.name) + except Exception as e: + logger.debug("unknown platform name %r: %s", entry.name, e) + return + existing_cfg = config.platforms.get(platform) + already_enabled = existing_cfg is not None and existing_cfg.enabled + existing_extra = dict(existing_cfg.extra or {}) if existing_cfg is not None else {} + # Never re-enable a platform the user explicitly disabled (marker set by the YAML loader). + if existing_cfg is not None and not already_enabled and existing_extra.get("_enabled_explicit", False): + return + seed = _plugin_probe_seed(entry) + # Only consult is_connected for platforms not already enabled by YAML/env. + if not already_enabled and entry.is_connected is not None and not _plugin_is_configured(entry, existing_extra, seed): + return + # Verify dependencies LAST — only for platforms already enabled or past the credential gate. + try: + deps_ok = bool(entry.check_fn()) + except Exception as e: + logger.debug("check_fn for %s raised: %s", entry.name, e) + deps_ok = False + if not deps_ok and entry.ensure_deps_fn is None: + return + platform_config = config.platforms.setdefault(platform, PlatformConfig()) + platform_config.enabled = True + if seed: + # Commit the env-seeded extras (reuse the probe result; don't call env_enablement_fn twice). + seed = dict(seed) + home = seed.pop("home_channel", None) + platform_config.extra.update(seed) + if isinstance(home, dict) and home.get("chat_id"): + platform_config.home_channel = HomeChannel( + platform=platform, chat_id=str(home["chat_id"]), name=str(home.get("name") or "Home"), + thread_id=str(home["thread_id"]) if home.get("thread_id") else None, + ) + + +def _enable_plugin_platforms_from_env(config: GatewayConfig) -> None: + """Registry-driven enable for plugin platforms (built-ins have rows in ``_ENV_STEPS``). + + Enabled when credentials are configured (``is_connected`` MUST gate: ``check_fn`` + alone would enable unconfigured platforms that retry-connect forever) and deps are + present (``check_fn``) or installable later by ``create_adapter()`` — never here: + installing in this sweep pip-installed SDKs on every load and boot-looped the app. """ try: from hermes_cli.plugins import discover_plugins discover_plugins() # idempotent from gateway.platform_registry import platform_registry for entry in platform_registry.plugin_entries(): - try: - platform = Platform(entry.name) - except Exception as e: - logger.debug("unknown platform name %r: %s", entry.name, e) - continue - existing_cfg = config.platforms.get(platform) - # Never re-enable a platform the user explicitly disabled (marker set by the YAML loader). - if ( - existing_cfg is not None - and not existing_cfg.enabled - and bool((existing_cfg.extra or {}).get("_enabled_explicit", False)) - ): - continue - # Seed candidate extras so plugins whose ``is_connected`` reads ``config.extra`` (Google Chat) - # see the same state they will after enablement. - seed_for_probe = None - if entry.env_enablement_fn is not None: - try: - seed_for_probe = entry.env_enablement_fn() - except Exception as e: - logger.debug("env_enablement_fn for %s raised: %s", entry.name, e) - seed_for_probe = None - has_seed = isinstance(seed_for_probe, dict) and bool(seed_for_probe) - - # Only consult is_connected for platforms not already enabled by YAML/env (keep that decision). - if (existing_cfg is None or not existing_cfg.enabled) and entry.is_connected is not None: - try: - # Probe with ``enabled=True``: we ask "would this be configured if enabled?" — - # some ``is_connected`` short-circuit on ``config.enabled`` being False. - probe_cfg = existing_cfg - if probe_cfg is None or not probe_cfg.enabled: - probe_cfg = PlatformConfig( - enabled=True, - extra=dict(existing_cfg.extra or {}) if existing_cfg is not None else {}, - ) - if has_seed: - # Transient view; never mutate ``existing_cfg`` for the probe. - probe_extra = dict(probe_cfg.extra or {}) - for k, v in seed_for_probe.items(): - if k != "home_channel": - probe_extra.setdefault(k, v) - probe_cfg = PlatformConfig(enabled=True, extra=probe_extra) - configured = bool(entry.is_connected(probe_cfg)) - except Exception as exc: - logger.debug("is_connected for %s raised: %s — skipping enablement", entry.name, exc) - configured = False - if not configured: - logger.debug( - "Plugin platform '%s' available but not configured " - "(is_connected returned False) — skipping enable", - entry.name, - ) - continue - # Verify dependencies LAST — only for platforms already enabled or past the credential gate. - try: - deps_ok = bool(entry.check_fn()) - except Exception as e: - logger.debug("check_fn for %s raised: %s", entry.name, e) - deps_ok = False - if not deps_ok and entry.ensure_deps_fn is None: - continue - platform_config = config.platforms.setdefault(platform, PlatformConfig()) - platform_config.enabled = True - # Commit the env-seeded extras (reuse the probe result; don't call env_enablement_fn twice). - if has_seed: - seed = dict(seed_for_probe) - home = seed.pop("home_channel", None) - platform_config.extra.update(seed) - if isinstance(home, dict) and home.get("chat_id"): - platform_config.home_channel = HomeChannel( - platform=platform, - chat_id=str(home["chat_id"]), - name=str(home.get("name") or "Home"), - thread_id=str(home["thread_id"]) if home.get("thread_id") else None, - ) + _enable_plugin_platform(config, entry) except Exception as e: logger.debug("Plugin platform enable pass failed: %s", e) def _relay(config: GatewayConfig) -> None: - """Relay (generic connector-fronted platform, EXPERIMENTAL). Enabled by GATEWAY_RELAY_URL (env) - or gateway.relay_url (config.yaml). The adapter dials OUT to the connector (no inbound port), - so it only needs Platform.RELAY present+enabled for start_gateway()'s connect loop. The - connected-checker keys on extra["relay_url"], so the URL is mirrored into extra here. + """Relay (connector-fronted, EXPERIMENTAL): enabled by GATEWAY_RELAY_URL or + gateway.relay_url; the URL is mirrored into extra["relay_url"] for the connected-checker. - Relay-exclusive: a GATEWAY_RELAY_URL env stamp marks a connector-fronted deployment where the - connector owns every platform connection; a directly-connected adapter in the same process - would be a second unmanaged ingress (duplicate deliveries, split sessions, a live socket that - disarms scale-to-zero). So the env stamp disables all other messaging platforms — even ones - explicitly enabled in config.yaml. Non-messaging surfaces (local, api_server, webhook — same - exclusion set as the scale-to-zero arm gate) are untouched; relay via gateway.relay_url only - keeps the old additive behavior. Opt-out GATEWAY_RELAY_ALLOW_DIRECT_PLATFORMS=true is likewise - a deploy-stamp env var read through the profile-scope-aware getenv. + Relay-exclusive: the GATEWAY_RELAY_URL env stamp means the connector owns every + platform connection; a directly-connected adapter would be a second unmanaged ingress + (duplicate deliveries, split sessions, a socket disarming scale-to-zero). So the env + stamp disables all other messaging platforms — even ones explicitly enabled — except + non-messaging surfaces (local, api_server, webhook; same set as the scale-to-zero arm + gate). relay_url from YAML only keeps the additive behavior. Opt out with + GATEWAY_RELAY_ALLOW_DIRECT_PLATFORMS=true. """ relay_url_env = getenv("GATEWAY_RELAY_URL").strip() existing_relay = config.platforms.get(Platform.RELAY) @@ -554,10 +505,9 @@ def _scrub_explicit_markers(config: GatewayConfig) -> None: platform_config.extra.pop("_enabled_explicit", None) -# Application order is significant: e.g. Telegram's reply mode may create the (disabled) platform -# entry BEFORE its home channel is read, while Discord reads home first; a home channel is only -# attached to a platform that already exists. Relay-exclusive disabling runs after the plugin -# pass and before the marker scrub, which must be last. +# Order is significant: a home channel only attaches to a platform that already exists +# (Telegram's reply mode may create the entry first; Discord reads home first). Relay +# disabling runs after the plugin pass; the marker scrub must be last. _ENV_STEPS: tuple = ( _Cred(Platform.TELEGRAM, ("TELEGRAM_BOT_TOKEN",), token="TELEGRAM_BOT_TOKEN"), _ReplyMode(Platform.TELEGRAM, "TELEGRAM_REPLY_TO_MODE"), @@ -568,29 +518,25 @@ _ENV_STEPS: tuple = ( _ReplyMode(Platform.DISCORD, "DISCORD_REPLY_TO_MODE"), _whatsapp, _Home(Platform.WHATSAPP, "WHATSAPP_HOME_CHANNEL"), - # WhatsApp Cloud API (official Business Platform via Meta). Distinct from the Baileys bridge; + # WhatsApp Cloud API (Meta Business Platform). Distinct from the Baileys bridge; # both adapters can run in parallel against different phone numbers. _Cred( Platform.WHATSAPP_CLOUD, ("WHATSAPP_CLOUD_PHONE_NUMBER_ID", "WHATSAPP_CLOUD_ACCESS_TOKEN"), fixed=(("phone_number_id", "WHATSAPP_CLOUD_PHONE_NUMBER_ID"), ("access_token", "WHATSAPP_CLOUD_ACCESS_TOKEN")), optional=( - ("app_id", "WHATSAPP_CLOUD_APP_ID"), - ("app_secret", "WHATSAPP_CLOUD_APP_SECRET"), - ("waba_id", "WHATSAPP_CLOUD_WABA_ID"), - ("verify_token", "WHATSAPP_CLOUD_VERIFY_TOKEN"), # Meta hub.verify_token shared secret - ("webhook_host", "WHATSAPP_CLOUD_WEBHOOK_HOST"), - ("webhook_port", "WHATSAPP_CLOUD_WEBHOOK_PORT", _INT), - ("webhook_path", "WHATSAPP_CLOUD_WEBHOOK_PATH"), - ("api_version", "WHATSAPP_CLOUD_API_VERSION"), + ("app_id", "WHATSAPP_CLOUD_APP_ID"), ("app_secret", "WHATSAPP_CLOUD_APP_SECRET"), + ("waba_id", "WHATSAPP_CLOUD_WABA_ID"), ("verify_token", "WHATSAPP_CLOUD_VERIFY_TOKEN"), + ("webhook_host", "WHATSAPP_CLOUD_WEBHOOK_HOST"), ("webhook_port", "WHATSAPP_CLOUD_WEBHOOK_PORT", _INT), + ("webhook_path", "WHATSAPP_CLOUD_WEBHOOK_PATH"), ("api_version", "WHATSAPP_CLOUD_API_VERSION"), ), ), _Home(Platform.WHATSAPP_CLOUD, "WHATSAPP_CLOUD_HOME_CHANNEL"), - _slack, + _Cred(Platform.SLACK, ("SLACK_BOT_TOKEN",), token="SLACK_BOT_TOKEN"), + _slack_home, _Cred( Platform.SIGNAL, ("SIGNAL_HTTP_URL", "SIGNAL_ACCOUNT"), fixed=( - ("http_url", "SIGNAL_HTTP_URL"), - ("account", "SIGNAL_ACCOUNT"), + ("http_url", "SIGNAL_HTTP_URL"), ("account", "SIGNAL_ACCOUNT"), ("ignore_stories", "SIGNAL_IGNORE_STORIES", "true", is_truthy_value), ), ), @@ -615,7 +561,7 @@ _ENV_STEPS: tuple = ( fixed=(("address", "EMAIL_ADDRESS"), ("imap_host", "EMAIL_IMAP_HOST"), ("smtp_host", "EMAIL_SMTP_HOST")), ), _Home(Platform.EMAIL, "EMAIL_HOME_ADDRESS"), - _sms, + _Cred(Platform.SMS, ("TWILIO_ACCOUNT_SID",), then=_sms_api_key), _Home(Platform.SMS, "SMS_HOME_CHANNEL"), _api_server, _webhook, @@ -628,10 +574,8 @@ _ENV_STEPS: tuple = ( _Cred( Platform.FEISHU, ("FEISHU_APP_ID", "FEISHU_APP_SECRET"), fixed=( - ("app_id", "FEISHU_APP_ID"), - ("app_secret", "FEISHU_APP_SECRET"), - ("domain", "FEISHU_DOMAIN", "feishu"), - ("connection_mode", "FEISHU_CONNECTION_MODE", "websocket"), + ("app_id", "FEISHU_APP_ID"), ("app_secret", "FEISHU_APP_SECRET"), + ("domain", "FEISHU_DOMAIN", "feishu"), ("connection_mode", "FEISHU_CONNECTION_MODE", "websocket"), ), optional=(("encrypt_key", "FEISHU_ENCRYPT_KEY"), ("verification_token", "FEISHU_VERIFICATION_TOKEN")), home="FEISHU_HOME_CHANNEL", @@ -645,15 +589,12 @@ _ENV_STEPS: tuple = ( _Cred( Platform.WECOM_CALLBACK, ("WECOM_CALLBACK_CORP_ID", "WECOM_CALLBACK_CORP_SECRET"), fixed=( - ("corp_id", "WECOM_CALLBACK_CORP_ID"), - ("corp_secret", "WECOM_CALLBACK_CORP_SECRET"), - ("agent_id", "WECOM_CALLBACK_AGENT_ID"), - ("token", "WECOM_CALLBACK_TOKEN"), + ("corp_id", "WECOM_CALLBACK_CORP_ID"), ("corp_secret", "WECOM_CALLBACK_CORP_SECRET"), + ("agent_id", "WECOM_CALLBACK_AGENT_ID"), ("token", "WECOM_CALLBACK_TOKEN"), ("encoding_aes_key", "WECOM_CALLBACK_ENCODING_AES_KEY"), - # No default: an unset WECOM_CALLBACK_HOST leaves extra.host falsy so the adapter's - # dual-stack DEFAULT_HOST=None applies (binds IPv4 + IPv6; "0.0.0.0" was IPv4-only). - ("host", "WECOM_CALLBACK_HOST"), - ("port", "WECOM_CALLBACK_PORT", "", _int_or(8645)), + # No host default: a falsy extra.host lets the adapter's dual-stack DEFAULT_HOST=None + # apply (binds IPv4 + IPv6; "0.0.0.0" was IPv4-only). + ("host", "WECOM_CALLBACK_HOST"), ("port", "WECOM_CALLBACK_PORT", "", _int_or(8645)), ), ), # Weixin (personal WeChat via iLink Bot API) @@ -661,12 +602,9 @@ _ENV_STEPS: tuple = ( Platform.WEIXIN, (("WEIXIN_TOKEN", "WEIXIN_ACCOUNT_ID"),), token="WEIXIN_TOKEN", optional=(("account_id", "WEIXIN_ACCOUNT_ID"),), optional_stripped=( - ("base_url", "WEIXIN_BASE_URL", _strip_slash), - ("cdn_base_url", "WEIXIN_CDN_BASE_URL", _strip_slash), - ("dm_policy", "WEIXIN_DM_POLICY", str.lower), - ("group_policy", "WEIXIN_GROUP_POLICY", str.lower), - ("allow_from", "WEIXIN_ALLOWED_USERS"), - ("group_allow_from", "WEIXIN_GROUP_ALLOWED_USERS"), + ("base_url", "WEIXIN_BASE_URL", _strip_slash), ("cdn_base_url", "WEIXIN_CDN_BASE_URL", _strip_slash), + ("dm_policy", "WEIXIN_DM_POLICY", str.lower), ("group_policy", "WEIXIN_GROUP_POLICY", str.lower), + ("allow_from", "WEIXIN_ALLOWED_USERS"), ("group_allow_from", "WEIXIN_GROUP_ALLOWED_USERS"), ("split_multiline_messages", "WEIXIN_SPLIT_MULTILINE_MESSAGES"), ), home="WEIXIN_HOME_CHANNEL", home_strip=True, @@ -675,8 +613,7 @@ _ENV_STEPS: tuple = ( _Cred( Platform.BLUEBUBBLES, ("BLUEBUBBLES_SERVER_URL", "BLUEBUBBLES_PASSWORD"), fixed=( - ("server_url", "BLUEBUBBLES_SERVER_URL", "", _strip_slash), - ("password", "BLUEBUBBLES_PASSWORD"), + ("server_url", "BLUEBUBBLES_SERVER_URL", "", _strip_slash), ("password", "BLUEBUBBLES_PASSWORD"), ("webhook_host", "BLUEBUBBLES_WEBHOOK_HOST", "127.0.0.1"), ("webhook_port", "BLUEBUBBLES_WEBHOOK_PORT", "", _int_or(8645)), ("webhook_path", "BLUEBUBBLES_WEBHOOK_PATH", "/bluebubbles-webhook"), @@ -698,14 +635,10 @@ _ENV_STEPS: tuple = ( Platform.YUANBAO, (("YUANBAO_APP_ID", "YUANBAO_APP_KEY"), "YUANBAO_APP_SECRET"), fixed=(("app_id", ("YUANBAO_APP_ID", "YUANBAO_APP_KEY")), ("app_secret", "YUANBAO_APP_SECRET")), optional=( - ("bot_id", "YUANBAO_BOT_ID"), - ("ws_url", "YUANBAO_WS_URL"), - ("api_domain", "YUANBAO_API_DOMAIN"), - ("route_env", "YUANBAO_ROUTE_ENV"), - ("dm_policy", "YUANBAO_DM_POLICY", lambda v: v.strip().lower()), - ("dm_allow_from", "YUANBAO_DM_ALLOW_FROM"), - ("group_policy", "YUANBAO_GROUP_POLICY", lambda v: v.strip().lower()), - ("group_allow_from", "YUANBAO_GROUP_ALLOW_FROM"), + ("bot_id", "YUANBAO_BOT_ID"), ("ws_url", "YUANBAO_WS_URL"), + ("api_domain", "YUANBAO_API_DOMAIN"), ("route_env", "YUANBAO_ROUTE_ENV"), + ("dm_policy", "YUANBAO_DM_POLICY", _strip_lower), ("dm_allow_from", "YUANBAO_DM_ALLOW_FROM"), + ("group_policy", "YUANBAO_GROUP_POLICY", _strip_lower), ("group_allow_from", "YUANBAO_GROUP_ALLOW_FROM"), ), home="YUANBAO_HOME_CHANNEL", ), diff --git a/gateway/config_loader.py b/gateway/config_loader.py index abec9891a2..6ef62d927b 100644 --- a/gateway/config_loader.py +++ b/gateway/config_loader.py @@ -62,31 +62,32 @@ def _is_dict(value: Any) -> bool: return isinstance(value, dict) +def _dm_behavior_choice(value: Any, default: str = "pair") -> str: + return _normalize_choice(value, {"pair", "ignore"}, default) + + +def _presence(*keys: str) -> tuple: + return tuple((k, k, "presence", None, None) for k in keys) + + # (yaml key, gw_data key, mode, accept(value) -> bool, transform(value)) _TOPLEVEL_BRIDGE: tuple = ( ("session_reset", "default_reset_policy", "presence", lambda v: bool(v) and isinstance(v, dict), None), ("quick_commands", "quick_commands", "none", _quick_commands_ok, None), ("stt", "stt", "presence", _is_dict, None), - ("stt_echo_transcripts", "stt_echo_transcripts", "presence", None, None), - ("group_sessions_per_user", "group_sessions_per_user", "presence", None, None), - ("thread_sessions_per_user", "thread_sessions_per_user", "presence", None, None), + *_presence("stt_echo_transcripts", "group_sessions_per_user", "thread_sessions_per_user"), ("multiplex_profiles", "multiplex_profiles", "gwdata", None, None), - ("multiplex_profile_allowlist", "multiplex_profile_allowlist", "presence", None, None), - ("room_link_url", "room_link_url", "presence", None, None), + *_presence("multiplex_profile_allowlist", "room_link_url"), ("profile_routes", "profile_routes", "none", lambda v: isinstance(v, list), None), - ("max_concurrent_sessions", "max_concurrent_sessions", "presence", None, None), + *_presence("max_concurrent_sessions"), ("systemd_watchdog_seconds", "systemd_watchdog_seconds", "nested", None, None), ("streaming", "streaming", "dict", None, None), - ("reset_triggers", "reset_triggers", "presence", None, None), - ("always_log_local", "always_log_local", "presence", None, None), - ("write_sessions_json", "write_sessions_json", "presence", None, None), - ("loop_watchdog", "loop_watchdog", "presence", None, None), - ("loop_watchdog_probe_interval_s", "loop_watchdog_probe_interval_s", "presence", None, None), - ("loop_watchdog_probe_timeout_s", "loop_watchdog_probe_timeout_s", "presence", None, None), - ("loop_watchdog_max_strikes", "loop_watchdog_max_strikes", "presence", None, None), - ("filter_silence_narration", "filter_silence_narration", "presence", None, None), - ("unauthorized_dm_behavior", "unauthorized_dm_behavior", "presence", None, - lambda v: _normalize_choice(v, {"pair", "ignore"}, "pair")), + *_presence( + "reset_triggers", "always_log_local", "write_sessions_json", "loop_watchdog", + "loop_watchdog_probe_interval_s", "loop_watchdog_probe_timeout_s", "loop_watchdog_max_strikes", + "filter_silence_narration", + ), + ("unauthorized_dm_behavior", "unauthorized_dm_behavior", "presence", None, _dm_behavior_choice), ) @@ -153,21 +154,10 @@ def merge_platform_sections(yaml_cfg: dict, gateway_cfg: Any, gw_data: dict) -> merged["extra"] = merged_extra platforms_data[plat_name] = merged - gateway_platforms = gateway_cfg.get("platforms") if isinstance(gateway_cfg, dict) else None - merge(gateway_platforms) + nested_gateway = gateway_cfg if isinstance(gateway_cfg, dict) else {} + merge(nested_gateway.get("platforms")) merge(yaml_cfg.get("platforms")) - if isinstance(gateway_cfg, dict): - nested: dict = {} - for key, value in gateway_cfg.items(): - if key == "platforms": - continue - try: - Platform(key) - except (ValueError, AttributeError): - continue - if isinstance(value, dict): - nested[key] = value - merge(nested) + merge({k: v for k, v in nested_gateway.items() if k != "platforms" and isinstance(v, dict) and _is_platform_name(k)}) api_plat = platforms_data.get("api_server") if isinstance(api_plat, dict): @@ -178,6 +168,14 @@ def merge_platform_sections(yaml_cfg: dict, gateway_cfg: Any, gw_data: dict) -> return platforms_data +def _is_platform_name(key: Any) -> bool: + try: + Platform(key) + except (ValueError, AttributeError): + return False + return True + + def platform_section(yaml_cfg: dict, name: str, gateway_platforms: Any) -> tuple: """Return ``(section, is_toplevel)`` for platform *name*. @@ -199,43 +197,32 @@ def _str_keyed(value: Any) -> Any: return {str(k): v for k, v in value.items()} if isinstance(value, dict) else value -def _dm_behavior(gw_data: dict): - return lambda v: _normalize_choice(v, {"pair", "ignore"}, gw_data.get("unauthorized_dm_behavior", "pair")) - - _TELEGRAM = frozenset({Platform.TELEGRAM}) _DISCORD_SLACK = frozenset({Platform.DISCORD, Platform.SLACK}) -# Keys copied from a platform's YAML section into ``extra`` (``None`` = every platform); -# third element transforms the value. +def _plain(*keys: str) -> tuple: + """Keys copied verbatim into ``extra`` for every platform.""" + return tuple((k, None, None) for k in keys) + + +# (key, platforms-or-None, transform-or-None), in ``extra`` insertion order. +# ``"dm"`` defers to the global unauthorized_dm_behavior. _SHARED_KEYS: tuple = ( ("unauthorized_dm_behavior", None, "dm"), ("notice_delivery", None, lambda v: _normalize_choice(v, {"public", "private"}, "public")), - ("reply_prefix", None, None), - ("reply_in_thread", None, None), - ("cron_continuable_surface", None, None), - ("require_mention", None, None), - ("send_read_receipts", None, None), + *_plain("reply_prefix", "reply_in_thread", "cron_continuable_surface", "require_mention", "send_read_receipts"), ("allowed_chats", _TELEGRAM, None), ("group_allowed_chats", _TELEGRAM, None), ("allowed_topics", _TELEGRAM, None), - ("free_response_channels", None, None), - ("mention_patterns", None, None), - ("exclusive_bot_mentions", None, None), + *_plain("free_response_channels", "mention_patterns", "exclusive_bot_mentions"), ("observe_unmentioned_group_messages", _TELEGRAM, None), - ("dm_policy", None, None), - ("allow_from", None, None), - ("allow_admin_from", None, None), - ("user_allowed_commands", None, None), - ("group_policy", None, None), - ("group_allow_from", None, None), - ("group_allow_admin_from", None, None), - ("group_user_allowed_commands", None, None), + *_plain( + "dm_policy", "allow_from", "allow_admin_from", "user_allowed_commands", + "group_policy", "group_allow_from", "group_allow_admin_from", "group_user_allowed_commands", + ), ("channel_skill_bindings", _DISCORD_SLACK, None), ("channel_prompts", None, _str_keyed), - ("gateway_restart_notification", None, None), - ("typing_indicator", None, None), - ("typing_status_text", None, None), + *_plain("gateway_restart_notification", "typing_indicator", "typing_status_text"), ) # Top-level port/host/secret bridged into ``extra`` for adapters that read them from config.extra. @@ -254,8 +241,9 @@ def _bridged_keys(plat: Platform, platform_cfg: dict, gw_data: dict) -> dict: if key not in platform_cfg or (only is not None and plat not in only): continue if transform == "dm": - transform = _dm_behavior(gw_data) - bridged[key] = transform(platform_cfg[key]) if transform else platform_cfg[key] + bridged[key] = _dm_behavior_choice(platform_cfg[key], gw_data.get("unauthorized_dm_behavior", "pair")) + else: + bridged[key] = transform(platform_cfg[key]) if transform else platform_cfg[key] for key in _PORT_BRIDGE_KEYS.get(plat, ()): if key in platform_cfg and key not in platform_cfg.get("extra", {}): bridged[key] = platform_cfg[key] @@ -285,7 +273,7 @@ def bridge_platform_shared_keys( ``merge_platform_sections`` already merged it with the correct precedence, and re-applying it here would overwrite that. An explicit top-level enable/disable sets ``_enabled_explicit`` so the env pass honors ``enabled: false`` for migrated plugin platforms instead of re-enabling them on - token/SDK presence (#41112). + token/SDK presence. """ for plat in targets: if plat == Platform.LOCAL: @@ -315,7 +303,7 @@ def bridge_platform_shared_keys( def apply_plugin_yaml_hooks(yaml_cfg: dict, gateway_platforms: Any, platforms_data: dict, registry) -> None: - """Plugin-owned YAML→env config bridges (``PlatformEntry.apply_yaml_config_fn``, #24836). + """Plugin-owned YAML→env config bridges (``PlatformEntry.apply_yaml_config_fn``). Order: shared-key loop → this dispatch → core-only bridges (require_mention/signal) → ``_apply_env_overrides()`` after ``GatewayConfig.from_dict``. @@ -333,20 +321,16 @@ def apply_plugin_yaml_hooks(yaml_cfg: dict, gateway_platforms: Any, platforms_da except Exception as e: logger.debug("apply_yaml_config_fn for %s raised: %s", entry.name, e) continue - if not isinstance(seeded, dict) or not seeded: - continue - _dict_slot(_dict_slot(platforms_data, entry.name), "extra").update(seeded) + if isinstance(seeded, dict) and seeded: + _dict_slot(_dict_slot(platforms_data, entry.name), "extra").update(seeded) def bridge_core_env_settings(yaml_cfg: dict, platforms_data: dict) -> None: - """The two YAML→env bridges that stay in core. - - Slack/Telegram/WhatsApp/DingTalk/Mattermost/Matrix/Feishu settings→env bridges migrated to - each plugin's ``apply_yaml_config_fn`` hook (#41112 / #3823 / #25443). + """The two YAML→env bridges that stay in core (the per-platform ones live in plugin hooks). Top-level ``require_mention`` → Telegram when the ``telegram:`` section has none: users write it - alongside ``group_sessions_per_user`` expecting it to work (#3979). It keys off the TOP-LEVEL key, - so the telegram plugin's hook (which only runs when a telegram block exists) can't cover it. + alongside ``group_sessions_per_user`` expecting it to work. It keys off the TOP-LEVEL key, so + the telegram plugin's hook (which only runs when a telegram block exists) can't cover it. Signal ``require_mention`` → ``SIGNAL_REQUIRE_MENTION`` (env wins when already set). """ tl_require_mention = yaml_cfg.get("require_mention") diff --git a/gateway/control_socket.py b/gateway/control_socket.py index 668ac22608..6a767a7a01 100644 --- a/gateway/control_socket.py +++ b/gateway/control_socket.py @@ -1,18 +1,10 @@ -"""Gateway control socket — the gateway-owned local coordination surface. - -Instead of process-table scans or ``gateway_state.json`` (which can outlive -its writer), the gateway serves a local-only socket answering versioned JSON -verbs (``identify``, ``status``). A connectable socket with a well-formed -``identify`` answer IS liveness — no PID-reuse heuristics. - -Transport — never a TCP port; filesystem/pipe ACLs are the auth boundary. -POSIX: ``$HERMES_HOME/gateway.sock``, or a temp-dir socket plus a -``gateway.sock.path`` pointer file when the home path exceeds ``sun_path``. -Windows: named pipe ``\\\\.\\pipe\\hermes-gateway-``. - -Wire contract: ONE request per connection — one JSON line in, one out, then -the server closes (no keep-alive/pipelining). Consumers PREFER the socket and -fall back to the state-file/scan layer when it doesn't answer. +"""Gateway control socket — the gateway-owned local coordination surface: a local-only socket answering +versioned JSON verbs (``identify``, ``status``). A connectable socket with a well-formed ``identify`` +answer IS liveness — no PID-reuse heuristics. Never a TCP port: filesystem/pipe ACLs are the auth +boundary. POSIX: ``$HERMES_HOME/gateway.sock`` (or a temp-dir socket + ``gateway.sock.path`` pointer +file when the home path exceeds ``sun_path``); Windows: named pipe ``\\\\.\\pipe\\hermes-gateway-``. +Wire contract: ONE request per connection — one JSON line in, one out, then the server closes. +Consumers PREFER the socket and fall back to the state-file/scan layer when it doesn't answer. """ from __future__ import annotations @@ -33,28 +25,18 @@ from typing import Any, Callable, Optional logger = logging.getLogger(__name__) CONTROL_PROTOCOL_VERSION = 1 - _SOCKET_FILENAME = "gateway.sock" _POINTER_FILENAME = "gateway.sock.path" _IS_WINDOWS = sys.platform == "win32" - -# sun_path limit is 104 on macOS/BSD, 108 on Linux; leave margin for the NUL. -_MAX_UNIX_PATH = 100 - +_MAX_UNIX_PATH = 100 # sun_path limit is 104 on macOS/BSD, 108 on Linux; margin for the NUL # Single-line JSON in/out; bounded so a misbehaving peer can't balloon memory. _MAX_REQUEST_BYTES = 64 * 1024 _MAX_RESPONSE_BYTES = 512 * 1024 - _DEFAULT_CLIENT_TIMEOUT = 2.0 -# --------------------------------------------------------------------------- -# Path resolution (shared by server and client) -# --------------------------------------------------------------------------- - def _home_hash(home: Path) -> str: - canonical = os.path.normcase(str(Path(home).expanduser().resolve(strict=False))) - return hashlib.sha256(canonical.encode("utf-8")).hexdigest()[:16] + return hashlib.sha256(os.path.normcase(str(Path(home).expanduser().resolve(strict=False))).encode("utf-8")).hexdigest()[:16] def windows_pipe_name(home: Path) -> str: @@ -67,29 +49,18 @@ def _fits_sun_path(path: Path) -> bool: def _fallback_socket_path(home: Path) -> Path: - """Short temp-dir path for homes whose direct socket path exceeds sun_path. - - Tries ``tempfile.gettempdir()`` then ``/tmp`` (POSIX). If nothing fits the - tempdir candidate is returned anyway — bind fails non-fatally and - consumers use the scan layer. - """ + """Short temp-dir path for homes whose direct socket path exceeds sun_path: ``tempfile.gettempdir()`` + then ``/tmp`` (POSIX); if nothing fits the tempdir candidate is returned anyway — bind fails + non-fatally and consumers use the scan layer.""" name = f"hermes-gw-{_home_hash(home)}.sock" - candidates = [Path(tempfile.gettempdir()) / name] - if not _IS_WINDOWS: - candidates.append(Path("/tmp") / name) - for candidate in candidates: - if _fits_sun_path(candidate): - return candidate - return candidates[0] + candidates = [Path(tempfile.gettempdir()) / name] + ([] if _IS_WINDOWS else [Path("/tmp") / name]) + return next((c for c in candidates if _fits_sun_path(c)), candidates[0]) def resolve_server_socket_path(home: Path) -> tuple[Path, Optional[Path]]: - """Return ``(bind_path, pointer_file)``; pointer_file is non-None only - when the temp-dir fallback is in use and the server must record it.""" + """Return ``(bind_path, pointer_file)``; pointer_file is set only for the temp-dir fallback.""" direct = Path(home) / _SOCKET_FILENAME - if _fits_sun_path(direct): - return direct, None - return _fallback_socket_path(home), Path(home) / _POINTER_FILENAME + return (direct, None) if _fits_sun_path(direct) else (_fallback_socket_path(home), Path(home) / _POINTER_FILENAME) def resolve_client_socket_path(home: Path) -> Optional[Path]: @@ -97,98 +68,60 @@ def resolve_client_socket_path(home: Path) -> Optional[Path]: direct = Path(home) / _SOCKET_FILENAME if direct.exists(): return direct - pointer = Path(home) / _POINTER_FILENAME - try: - if pointer.is_file(): - target = pointer.read_text(encoding="utf-8").strip() - if target and Path(target).exists(): - return Path(target) - except OSError: - pass + with contextlib.suppress(OSError): + pointer = Path(home) / _POINTER_FILENAME + target = pointer.read_text(encoding="utf-8").strip() if pointer.is_file() else "" + if target and Path(target).exists(): + return Path(target) return None -# --------------------------------------------------------------------------- -# Default payload builders (import-light; overridable at wiring time) -# --------------------------------------------------------------------------- - def _detect_supervisor() -> str: - """Supervisor kind for THIS process, declared from its own launch env - (not inferred outside-in like ``_detect_supervisor_for_pid``).""" + """Supervisor kind for THIS process from its own launch env (not inferred outside-in).""" env = os.environ if env.get("INVOCATION_ID"): return "systemd" - if sys.platform == "darwin" and ( - env.get("XPC_SERVICE_NAME", "").startswith("ai.hermes") - or env.get("LAUNCHD_SOCKET") - ): + if sys.platform == "darwin" and (env.get("XPC_SERVICE_NAME", "").startswith("ai.hermes") + or env.get("LAUNCHD_SOCKET")): return "launchd" if env.get("HERMES_DESKTOP_MANAGED"): return "desktop" - if "--external-supervisor" in sys.argv: - return "external" - return "manual" + return "external" if "--external-supervisor" in sys.argv else "manual" def build_identify_payload() -> dict[str, Any]: """Default ``identify`` answer, built from gateway.status primitives.""" - from gateway.status import ( - _build_pid_record, - _get_code_identity_fields, - _profile_label_for_home, - read_runtime_status, - ) - + from gateway.status import _build_pid_record, _get_code_identity_fields, _profile_label_for_home, read_runtime_status record = _build_pid_record() payload: dict[str, Any] = { "protocol": CONTROL_PROTOCOL_VERSION, **{k: record.get(k) for k in ("kind", "pid", "start_time", "hermes_home")}, "profile": _profile_label_for_home(record.get("hermes_home") or ""), - "supervisor": _detect_supervisor(), - } - payload.update(_get_code_identity_fields()) - # served_profiles (multiplex mode) is stamped into runtime status by the runner. - try: + "supervisor": _detect_supervisor(), **_get_code_identity_fields()} + with contextlib.suppress(Exception): + # served_profiles (multiplex mode) is stamped into runtime status by the runner. served = (read_runtime_status() or {}).get("served_profiles") if isinstance(served, list) and served: payload["served_profiles"] = served - except Exception: - pass return payload def build_status_payload() -> dict[str, Any]: """Default ``status`` answer — current runtime status, answered live.""" from gateway.status import read_runtime_status + return {**(read_runtime_status() or {}), "protocol": CONTROL_PROTOCOL_VERSION, + "answered_at": time.time(), "answering_pid": os.getpid()} - payload = dict(read_runtime_status() or {}) - payload["protocol"] = CONTROL_PROTOCOL_VERSION - payload["answered_at"] = time.time() - payload["answering_pid"] = os.getpid() - return payload - - -# --------------------------------------------------------------------------- -# Server -# --------------------------------------------------------------------------- class GatewayControlServer: - """Gateway-owned control socket server (identify/status, v1). + """Gateway-owned control socket server (identify/status, v1): ``start()`` after the PID-file claim, + ``stop()`` on shutdown. All failures are non-fatal — the gateway never refuses to serve messaging + because its control socket couldn't bind; consumers fall back to the scan layer.""" - ``start()`` after the PID-file claim, ``stop()`` on shutdown. All failures - are non-fatal — the gateway never refuses to serve messaging because its - control socket couldn't bind; consumers fall back to the scan layer. - """ - - def __init__( - self, - home: Optional[Path] = None, - *, - verb_handlers: Optional[dict[str, Callable[[], dict[str, Any]]]] = None, - ) -> None: + def __init__(self, home: Optional[Path] = None, *, + verb_handlers: Optional[dict[str, Callable[[], dict[str, Any]]]] = None) -> None: if home is None: from gateway.status import _get_process_hermes_home - home = _get_process_hermes_home() self._home = Path(home) self._server: Optional[asyncio.AbstractServer] = None @@ -196,33 +129,24 @@ class GatewayControlServer: self._bind_path: Optional[Path] = None self._pointer_file: Optional[Path] = None self._handlers: dict[str, Callable[[], dict[str, Any]]] = { - "identify": build_identify_payload, - "status": build_status_payload, - } - if verb_handlers: - self._handlers.update(verb_handlers) - - # -- lifecycle --------------------------------------------------------- + "identify": build_identify_payload, "status": build_status_payload, **(verb_handlers or {})} async def start(self) -> bool: """Bind and start serving. Returns True on success, False otherwise.""" try: - if _IS_WINDOWS: - return await self._start_windows() - return await self._start_posix() + return await (self._start_windows() if _IS_WINDOWS else self._start_posix()) except Exception as exc: logger.warning("Gateway control socket failed to start (non-fatal): %s", exc) return False async def _start_posix(self) -> bool: bind_path, pointer_file = resolve_server_socket_path(self._home) - # We only get here after winning the PID-file O_EXCL race, so any - # existing file is stale or a collision — never a live sibling. + # We only get here after winning the PID-file O_EXCL race, so any existing + # file is stale or a collision — never a live sibling. with contextlib.suppress(OSError): if bind_path.exists(): bind_path.unlink() - # Restrictive umask so the socket is never world-connectable, even - # for the instant before chmod. + # Restrictive umask so the socket is never world-connectable, even for the instant before chmod. old_umask = os.umask(0o177) try: self._server = await asyncio.start_unix_server(self._handle_connection, path=str(bind_path)) @@ -241,10 +165,8 @@ class GatewayControlServer: loop = asyncio.get_running_loop() start_serving_pipe = getattr(loop, "start_serving_pipe", None) if start_serving_pipe is None: - logger.debug( - "Event loop %s has no start_serving_pipe — control socket " - "disabled (selector loop on Windows).", type(loop).__name__, - ) + logger.debug("Event loop %s has no start_serving_pipe — control socket " + "disabled (selector loop on Windows).", type(loop).__name__) return False pipe_name = windows_pipe_name(self._home) servers = await start_serving_pipe(lambda: _PipeControlProtocol(self), pipe_name) @@ -258,48 +180,34 @@ class GatewayControlServer: self._server.close() with contextlib.suppress(Exception): await self._server.wait_closed() - self._server = None if self._pipe_server is not None: with contextlib.suppress(Exception): self._pipe_server.close() - self._pipe_server = None + self._server = self._pipe_server = None self.cleanup_files() def cleanup_files(self) -> None: """Best-effort removal of socket + pointer files (atexit-safe).""" - for path in (self._bind_path, self._pointer_file): - if path is not None: - with contextlib.suppress(OSError): - path.unlink(missing_ok=True) - - # -- request handling ---------------------------------------------------- + for path in filter(None, (self._bind_path, self._pointer_file)): + with contextlib.suppress(OSError): + path.unlink(missing_ok=True) def handle_request_line(self, raw: bytes) -> bytes: - """One JSON request line -> one JSON response line. Never raises. - - Shared by the POSIX stream handler and the Windows pipe protocol. - """ + """One JSON request line -> one JSON response line. Never raises (shared by POSIX + pipe).""" request_id: Any = None try: request = json.loads(raw.decode("utf-8")) if not isinstance(request, dict): raise ValueError("request must be a JSON object") - request_id = request.get("id") - verb = request.get("verb") + request_id, verb = request.get("id"), request.get("verb") handler = self._handlers.get(verb) if isinstance(verb, str) else None if handler is None: - response: dict[str, Any] = { - "ok": False, "error": f"unknown verb: {verb!r}", - "protocol": CONTROL_PROTOCOL_VERSION, - "supported_verbs": sorted(self._handlers), - } + response: dict[str, Any] = {"ok": False, "error": f"unknown verb: {verb!r}", + "protocol": CONTROL_PROTOCOL_VERSION, "supported_verbs": sorted(self._handlers)} else: response = {"ok": True, "protocol": CONTROL_PROTOCOL_VERSION, "result": handler()} except Exception as exc: - response = { - "ok": False, "error": f"{type(exc).__name__}: {exc}", - "protocol": CONTROL_PROTOCOL_VERSION, - } + response = {"ok": False, "error": f"{type(exc).__name__}: {exc}", "protocol": CONTROL_PROTOCOL_VERSION} if request_id is not None: response["id"] = request_id try: @@ -315,10 +223,10 @@ class GatewayControlServer: raw = await asyncio.wait_for(reader.readline(), timeout=_DEFAULT_CLIENT_TIMEOUT) if not raw or len(raw) > _MAX_REQUEST_BYTES: return - # Handlers read disk; keep that off the loop that drives every - # platform adapter so a fast-polling consumer can't stall heartbeats. - loop = asyncio.get_running_loop() - response = await loop.run_in_executor(None, self.handle_request_line, raw.rstrip(b"\n")) + # Handlers read disk; keep that off the loop that drives every platform + # adapter so a fast-polling consumer can't stall heartbeats. + response = await asyncio.get_running_loop().run_in_executor( + None, self.handle_request_line, raw.rstrip(b"\n")) writer.write(response) await writer.drain() except (asyncio.TimeoutError, ConnectionError, OSError): @@ -332,7 +240,6 @@ class GatewayControlServer: class _PipeControlProtocol(asyncio.Protocol): """One-shot request/response protocol for the Windows named pipe.""" - def __init__(self, server: GatewayControlServer) -> None: self._server = server self._transport: Any = None @@ -345,43 +252,25 @@ class _PipeControlProtocol(asyncio.Protocol): self._buffer.extend(data) if len(self._buffer) > _MAX_REQUEST_BYTES: self._transport.close() - return - if b"\n" in self._buffer: - line, _, _ = bytes(self._buffer).partition(b"\n") + elif b"\n" in self._buffer: try: - self._transport.write(self._server.handle_request_line(line)) + self._transport.write(self._server.handle_request_line(bytes(self._buffer).partition(b"\n")[0])) finally: self._transport.close() -# --------------------------------------------------------------------------- -# Client (synchronous — used by CLI/updater consumers) -# --------------------------------------------------------------------------- - -def query_gateway_control( - home: Path, verb: str, *, timeout: float = _DEFAULT_CLIENT_TIMEOUT -) -> Optional[dict[str, Any]]: - """Ask the gateway serving ``home`` a control verb; None when unanswered. - - Returns the verb's ``result`` payload. Any failure (no/stale socket, - timeout, malformed answer, ``ok: false``) returns None so callers fall - back to the scan layer. Never raises. - """ +def query_gateway_control(home: Path, verb: str, *, timeout: float = _DEFAULT_CLIENT_TIMEOUT) -> Optional[dict[str, Any]]: + """Ask the gateway serving ``home`` a control verb; returns its ``result`` payload. Any failure (no/stale + socket, timeout, malformed answer, ``ok: false``) returns None so callers fall back to the scan layer. + Never raises.""" request = json.dumps({"verb": verb, "id": 1, "protocol": CONTROL_PROTOCOL_VERSION}).encode("utf-8") + b"\n" query = _query_windows_pipe if _IS_WINDOWS else _query_unix_socket try: raw = query(Path(home), request, timeout) + response = json.loads(raw.decode("utf-8")) if raw else None except Exception: return None - if not raw: - return None - try: - response = json.loads(raw.decode("utf-8")) - except (ValueError, UnicodeDecodeError): - return None - if not isinstance(response, dict) or response.get("ok") is not True: - return None - result = response.get("result") + result = response.get("result") if isinstance(response, dict) and response.get("ok") is True else None return result if isinstance(result, dict) else None @@ -390,32 +279,25 @@ def _read_response_line(read: Callable[[], bytes], deadline: float) -> Optional[ chunks: list[bytes] = [] while time.monotonic() < deadline: chunk = read() - if not chunk: - break chunks.append(chunk) - if b"\n" in chunk: + if not chunk or b"\n" in chunk: break if sum(len(c) for c in chunks) > _MAX_RESPONSE_BYTES: return None - line, _, _ = b"".join(chunks).partition(b"\n") - return line or None + return b"".join(chunks).partition(b"\n")[0] or None def _query_unix_socket(home: Path, request: bytes, timeout: float) -> Optional[bytes]: path = resolve_client_socket_path(home) if path is None: return None - with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as sock: + # OSError covers ConnectionRefusedError / FileNotFoundError on connect and socket.timeout on read. + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as sock, contextlib.suppress(OSError): sock.settimeout(timeout) - try: - sock.connect(str(path)) - except (ConnectionRefusedError, FileNotFoundError, OSError): - return None + sock.connect(str(path)) sock.sendall(request) - try: - return _read_response_line(lambda: sock.recv(65536), time.monotonic() + timeout) - except socket.timeout: - return None + return _read_response_line(lambda: sock.recv(65536), time.monotonic() + timeout) + return None def _query_windows_pipe(home: Path, request: bytes, timeout: float) -> Optional[bytes]: # pragma: no cover - wine2e lane @@ -446,10 +328,7 @@ def identify_gateway(home: Path, *, timeout: float = _DEFAULT_CLIENT_TIMEOUT) -> def pause_gateway_for_update(home: Path, *, timeout: float = _DEFAULT_CLIENT_TIMEOUT) -> Optional[dict[str, Any]]: - """Ask the gateway serving ``home`` to drain and exit for an update. - - Returns the ACK ``{"pausing", "already_stopping", "pid", "drain_timeout"}`` - or None when no gateway answers (old gateway without the verb, no/dead - socket) — the caller then uses the legacy signal/tree-kill pause path. - """ + """Ask the gateway serving ``home`` to drain and exit for an update. Returns the ACK ``{"pausing", + "already_stopping", "pid", "drain_timeout"}`` or None when no gateway answers (old gateway without + the verb, no/dead socket) — the caller then uses the legacy signal/tree-kill pause path.""" return query_gateway_control(home, "pause-for-update", timeout=timeout) diff --git a/gateway/cwd_placeholder.py b/gateway/cwd_placeholder.py index b064104c6a..33c1c418dd 100644 --- a/gateway/cwd_placeholder.py +++ b/gateway/cwd_placeholder.py @@ -22,16 +22,12 @@ def resolve_placeholder_terminal_cwd( ) -> str | None: """Return the ``TERMINAL_CWD`` value to set, or ``None`` to leave it unset. - Cases: - - **local** + placeholder → ``MESSAGING_CWD`` or ``home_fallback`` - - **docker** + placeholder + mount on + host ``MESSAGING_CWD`` → host path - (for ``terminal_tool`` ``/workspace`` mapping) - - **docker** + placeholder + mount off → ``None`` (sandbox default) - - other non-local backends + placeholder → ``None`` + local + placeholder → ``MESSAGING_CWD`` or ``home_fallback``; docker + + placeholder + mount on + host ``MESSAGING_CWD`` → that host path (for the + ``/workspace`` mapping); any other non-local backend → ``None`` (sandbox default). """ if configured_cwd and configured_cwd not in CWD_PLACEHOLDERS: return configured_cwd - backend = (terminal_backend or "local").strip().lower() messaging = (messaging_cwd or "").strip() if backend == "local": diff --git a/gateway/dead_targets.py b/gateway/dead_targets.py index b0ffa3cdd6..2b47b3083b 100644 --- a/gateway/dead_targets.py +++ b/gateway/dead_targets.py @@ -1,18 +1,9 @@ -"""Persistent registry of delivery targets that are confirmed unreachable. - -When a platform reports a target chat is permanently gone (deleted group, -bot kicked/blocked, deactivated user), re-sending on every cron tick wastes a -send against flood control and spams logs. The delivery layer short-circuits -targets proven dead; any later successful send clears the flag (self-healing). - -Scope is deliberately narrow: only *whole-chat* deaths (``forbidden`` and -chat-level ``not_found``) are recorded. Thread/topic-level ``not_found`` is -NOT — adapters self-heal that by retrying without ``reply_to``, and a deleted -topic does not mean the parent chat is dead. - -Storage is a JSON file under the active profile's HERMES_HOME (each profile -keeps its own dead set). Reads/writes are best-effort: a corrupt or unwritable -file degrades to in-memory-only rather than raising on the delivery path. +"""Persistent registry of delivery targets confirmed unreachable. Re-sending to a permanently gone chat +(deleted group, bot kicked/blocked, deactivated user) every cron tick wastes flood-control budget; delivery +short-circuits targets proven dead and any later successful send clears the flag. Only *whole-chat* deaths +(``forbidden``, chat-level ``not_found``) are recorded: adapters self-heal thread/topic-level ``not_found`` +by retrying without ``reply_to``. Storage is a per-profile JSON file; reads/writes are best-effort (a +corrupt or unwritable file degrades to in-memory-only rather than raising on the delivery path). """ from __future__ import annotations @@ -28,8 +19,7 @@ from hermes_cli.config import get_hermes_home logger = logging.getLogger(__name__) -# Error kinds (gateway.platforms.base.classify_send_error) meaning the whole -# chat is unreachable, not a transient or thread-level problem. +# classify_send_error kinds meaning the whole chat is unreachable (not transient / thread-level). _DEAD_ERROR_KINDS = frozenset({"forbidden", "not_found"}) @@ -38,29 +28,35 @@ def _normalize(platform: str, chat_id: str) -> str: return f"{str(platform).strip().lower()}:{str(chat_id).strip()}" -class DeadTargetRegistry: - """Thread-safe, persistent set of confirmed-dead targets keyed ``platform:chat_id``. +def classify_dead_error(error_text: Optional[str]) -> Optional[str]: + """Best-effort dead-target error_kind from a raised error's text, else None. ``_deliver_to_platform`` + raises on hard failure (no SendResult), so ``deliver()`` only has the exception string. ``not_found`` + collapses chat-level and thread/topic/message-level failures: only a whole-chat not_found means the + target is dead — a deleted forum topic or edited-away message must not mark the entire chat dead.""" + if not error_text: + return None + try: + from .platforms.base import classify_send_error, is_chat_level_not_found + except Exception: # pragma: no cover - import guard + return None + kind = classify_send_error(None, error_text=error_text) + dead = kind in _DEAD_ERROR_KINDS and (kind != "not_found" or is_chat_level_not_found(error_text=error_text)) + return kind if dead else None - Each entry stores reason + timestamp for observability; :meth:`clear` - (called on a successful send) removes the flag. - """ + +class DeadTargetRegistry: + """Thread-safe, persistent set of confirmed-dead targets keyed ``platform:chat_id``. Each entry stores + reason + timestamp for observability; :meth:`clear` (called on a successful send) removes the flag.""" def __init__(self, path: Optional[Path] = None) -> None: self._lock = threading.RLock() self._dead: Dict[str, Dict[str, object]] = {} self._path = path if path is not None else get_hermes_home() / "gateway" / "dead_targets.json" - self._load() - - def _load(self) -> None: try: - if self._path.exists(): - raw = json.loads(self._path.read_text(encoding="utf-8")) - if isinstance(raw, dict): - self._dead = {k: v for k, v in raw.items() if isinstance(v, dict)} + raw = json.loads(self._path.read_text(encoding="utf-8")) if self._path.exists() else {} + self._dead = {k: v for k, v in raw.items() if isinstance(v, dict)} if isinstance(raw, dict) else {} except (OSError, ValueError) as exc: - logger.debug("dead_targets: could not load %s (%s) — starting empty", - self._path, exc) - self._dead = {} + logger.debug("dead_targets: could not load %s (%s) — starting empty", self._path, exc) def _flush_locked(self) -> None: try: @@ -68,42 +64,26 @@ class DeadTargetRegistry: tmp = self._path.with_suffix(self._path.suffix + ".tmp") tmp.write_text(json.dumps(self._dead, indent=2), encoding="utf-8") tmp.replace(self._path) - except OSError as exc: - # Best-effort: keep in-memory state, never break delivery. + except OSError as exc: # best-effort: keep in-memory state, never break delivery logger.debug("dead_targets: could not persist %s (%s)", self._path, exc) - @staticmethod - def is_dead_error_kind(error_kind: Optional[str]) -> bool: - """True when ``error_kind`` denotes a permanent whole-chat death.""" - return bool(error_kind) and error_kind in _DEAD_ERROR_KINDS - def is_dead(self, platform: str, chat_id: Optional[str]) -> bool: - if not chat_id: - return False with self._lock: - return _normalize(platform, chat_id) in self._dead + return bool(chat_id) and _normalize(platform, chat_id) in self._dead - def mark_dead(self, platform: str, chat_id: Optional[str], - reason: str = "") -> bool: + def mark_dead(self, platform: str, chat_id: Optional[str], reason: str = "") -> bool: """Record a target as confirmed-dead. Returns True if newly added.""" if not chat_id: return False key = _normalize(platform, chat_id) with self._lock: existed = key in self._dead - self._dead[key] = { - "platform": str(platform).strip().lower(), - "chat_id": str(chat_id), - "reason": str(reason)[:200], - "marked_at": time.time(), - } + self._dead[key] = {"platform": str(platform).strip().lower(), "chat_id": str(chat_id), + "reason": str(reason)[:200], "marked_at": time.time()} self._flush_locked() if not existed: - logger.info( - "dead_targets: marked %s as unreachable (%s) — future deliveries " - "to this target will be skipped until a send succeeds", - key, reason or "no reason given", - ) + logger.info("dead_targets: marked %s as unreachable (%s) — future deliveries " + "to this target will be skipped until a send succeeds", key, reason or "no reason given") return not existed def clear(self, platform: str, chat_id: Optional[str]) -> bool: @@ -112,9 +92,8 @@ class DeadTargetRegistry: return False key = _normalize(platform, chat_id) with self._lock: - if key in self._dead: - del self._dead[key] - self._flush_locked() - logger.info("dead_targets: cleared %s (delivery succeeded again)", key) - return True - return False + if self._dead.pop(key, None) is None: + return False + self._flush_locked() + logger.info("dead_targets: cleared %s (delivery succeeded again)", key) + return True diff --git a/gateway/delivery.py b/gateway/delivery.py index e5ded9c5f8..b8478da44f 100644 --- a/gateway/delivery.py +++ b/gateway/delivery.py @@ -1,10 +1,5 @@ -""" -Delivery routing for cron job outputs and agent responses. - -Routes messages by target: explicit ("telegram:123456789"), platform home -channel ("telegram"), origin (back to where the job was created), or local -(saved to files). -""" +"""Delivery routing for cron job outputs and agent responses, by target: explicit ("telegram:123456789"), +platform home channel ("telegram"), origin (back to where the job was created), or local (files).""" import logging import os @@ -16,43 +11,35 @@ from typing import Dict, List, Optional, Any from hermes_cli.config import get_hermes_home +from .config import Platform, GatewayConfig, PlatformConfig +from .session import SessionSource +from .dead_targets import DeadTargetRegistry, classify_dead_error + logger = logging.getLogger(__name__) -# Cap before gateway-level truncation of cron output for non-chunking platform -# delivery. Telegram's hard API limit is 4096; the headroom covers the "full -# output saved to …" footer. Adapters that split long messages natively -# (BasePlatformAdapter.splits_long_messages) bypass this entirely. +# Cap before gateway-level truncation of cron output for non-chunking platform delivery. Telegram's hard +# API limit is 4096; the headroom covers the "full output saved to …" footer. Adapters that split long +# messages natively (splits_long_messages) bypass this entirely. MAX_PLATFORM_OUTPUT = 4000 - -# Matches strings that are *only* a "silence" narration with optional markdown -# wrappers: *(silent)*, _silent_, `silent`, (silent), silent, 🔇, a bare ".", -# "…", and marker-padded variants. Anchored so substantive messages that merely -# *contain* the word "silent" never match. +# Matches strings that are *only* a "silence" narration with optional markdown wrappers (*(silent)*, +# _silent_, 🔇, a bare ".", "…"). Anchored so messages that merely *contain* "silent" never match. _SILENCE_NARRATION = re.compile( r'^[\s*_~`]*\(?\s*(silent|silence|no\s+response|no\s+reply)\s*\.?\)?[\s*_~`]*$' r'|^[\s*_~`]*[\U0001F507\.\u2026]+[\s*_~`]*$', re.IGNORECASE, ) +_THREAD_ROUTING_KEYS = ("thread_id", "message_thread_id", "direct_messages_topic_id", "telegram_direct_messages_topic_id") def _is_silence_narration(content: Optional[str]) -> bool: """True when ``content`` is *only* a silence-narration token (length-guarded).""" - if not content: - return False - stripped = content.strip() - if not stripped or len(stripped) > 64: - return False - return bool(_SILENCE_NARRATION.match(stripped)) - -from .config import Platform, GatewayConfig, PlatformConfig -from .session import SessionSource -from .dead_targets import DeadTargetRegistry + stripped = content.strip() if content else "" + return bool(stripped) and len(stripped) <= 64 and bool(_SILENCE_NARRATION.match(stripped)) @dataclass(frozen=True) class DeliveryTransport: """Resolved live transport for one logical delivery platform.""" - adapter: Any config: Optional[PlatformConfig] transport_platform: Platform @@ -61,128 +48,59 @@ class DeliveryTransport: def is_relay(self) -> bool: return self.transport_platform == Platform.RELAY - async def send( - self, - logical_platform: Platform, - chat_id: str, - content: str, - metadata: Optional[Dict[str, Any]], - ) -> Any: + async def send(self, logical_platform: Platform, chat_id: str, content: str, + metadata: Optional[Dict[str, Any]]) -> Any: """Send through this transport while preserving the logical platform.""" - if self.is_relay: - return await self.adapter.send_for_platform( - logical_platform, chat_id, content, metadata=metadata, - ) - return await self.adapter.send(chat_id, content, metadata=metadata) + return await (self.adapter.send_for_platform(logical_platform, chat_id, content, metadata=metadata) + if self.is_relay else self.adapter.send(chat_id, content, metadata=metadata)) -def resolve_delivery_transport( - platform: Platform, - config: GatewayConfig, - adapters: Optional[Dict[Platform, Any]], -) -> Optional[DeliveryTransport]: - """Resolve a logical platform to its live delivery transport. - - A concrete native adapter always wins. Relay is eligible only when its - authenticated transport explicitly advertises that it fronts the logical - platform, so restart-time delivery is independent of per-chat caches without - letting Relay hijack unrelated platform targets. - """ +def resolve_delivery_transport(platform: Platform, config: GatewayConfig, + adapters: Optional[Dict[Platform, Any]]) -> Optional[DeliveryTransport]: + """Resolve a logical platform to its live delivery transport. A concrete native adapter always wins; + Relay is eligible only when its authenticated transport explicitly advertises that it fronts the + logical platform, so restart-time delivery is independent of per-chat caches without letting Relay + hijack unrelated platform targets.""" live_adapters = adapters or {} - native = live_adapters.get(platform) - native_config = config.platforms.get(platform) + native, native_config = live_adapters.get(platform), config.platforms.get(platform) # Explicitly supplied live adapters with no config block are honored, but an # explicitly disabled native adapter never shadows an enabled Relay transport. if native is not None and (native_config is None or native_config.enabled): return DeliveryTransport(native, native_config, platform) - - relay = live_adapters.get(Platform.RELAY) - relay_config = config.platforms.get(Platform.RELAY) + relay, relay_config = live_adapters.get(Platform.RELAY), config.platforms.get(Platform.RELAY) fronts_platform = getattr(relay, "fronts_platform", None) - if ( - relay is not None - and (relay_config is None or relay_config.enabled) - and callable(fronts_platform) - and fronts_platform(platform) - ): + if (relay is not None and (relay_config is None or relay_config.enabled) + and callable(fronts_platform) and fronts_platform(platform)): return DeliveryTransport(relay, relay_config, Platform.RELAY) return None -def _as_int(value: Any) -> Optional[int]: - try: - return int(value) - except (TypeError, ValueError): - return None - - def looks_like_telegram_private_chat_id(chat_id: Optional[str]) -> bool: - """True when ``chat_id`` is a positive int — Telegram's private-chat shape. - - Groups/channels/supergroups use negative IDs. Single source of truth for the - heuristic, reused by the handoff seed path in ``gateway/run.py`` so - handoff-created DM topics key the same way as inbound DM-topic messages. - """ - if chat_id is None: + """True when ``chat_id`` is a positive int — Telegram's private-chat shape (groups/channels are negative). + Single source of truth, reused by the handoff seed path in ``gateway/run.py`` so handoff-created DM + topics key the same way as inbound DM-topic messages.""" + try: + return int(chat_id) > 0 + except (TypeError, ValueError): return False - parsed = _as_int(chat_id) - return parsed is not None and parsed > 0 def _looks_like_int(value: Optional[str]) -> bool: - return value is not None and _as_int(value) is not None - - -def _result_field(result: Any, name: str, default: Any = None) -> Any: - """Read ``name`` from a SendResult object or a plain result dict.""" - if isinstance(result, dict): - return result.get(name, default) - return getattr(result, name, default) - - -def _send_result_failed(result: Any) -> bool: - return _result_field(result, "success", True) is False + try: + return int(value) is not None + except (TypeError, ValueError): + return False def _send_result_error(result: Any) -> Optional[str]: - error = _result_field(result, "error") - return str(error) if error else None - - -def _is_thread_not_found_delivery_error(result: Any) -> bool: - error = _send_result_error(result) - return bool(error and "thread not found" in error.lower()) - - -def _classify_dead_from_error_text(error_text: Optional[str]) -> Optional[str]: - """Best-effort dead-target classification from a raised error's text. - - ``_deliver_to_platform`` raises on hard failure (no SendResult), so the - ``deliver()`` loop only has the exception string; reuse the platform-neutral - classifier to recover the error_kind from it. - """ - if not error_text: - return None - try: - from .platforms.base import classify_send_error, is_chat_level_not_found - except Exception: # pragma: no cover - import guard - return None - kind = classify_send_error(None, error_text=error_text) - if not DeadTargetRegistry.is_dead_error_kind(kind): - return None - # ``not_found`` collapses chat-level and thread/topic/message-level failures. - # Only a whole-chat not_found means the target is dead — a deleted forum - # topic or an edited-away message must not mark the entire chat (and all its - # future deliveries) dead (see gateway.dead_targets). - if kind == "not_found" and not is_chat_level_not_found(error_text=error_text): - return None - return kind + """Error string of a failed SendResult object / plain result dict ("" if none), or None on success.""" + get = result.get if isinstance(result, dict) else (lambda name, default=None: getattr(result, name, default)) + return None if get("success", True) is not False else str(get("error") or "") @dataclass class DeliveryTarget: - """A single delivery target: "origin", "local", "telegram" (home channel), - or "telegram:123456[:thread]" (specific chat).""" + """One target: "origin", "local", "telegram" (home channel) or "telegram:123456[:thread]".""" platform: Platform chat_id: Optional[str] = None # None means use home channel thread_id: Optional[str] = None @@ -192,38 +110,18 @@ class DeliveryTarget: @classmethod def parse(cls, target: str, origin: Optional[SessionSource] = None) -> "DeliveryTarget": """Parse "origin" | "local" | "" | ":[:]".""" - target_stripped = target.strip() - target_lower = target_stripped.lower() - - if target_lower == "origin": - if origin: - return cls( - platform=origin.platform, - chat_id=origin.chat_id, - thread_id=origin.thread_id, - is_origin=True, - ) - return cls(platform=Platform.LOCAL, is_origin=True) - - if target_lower == "local": - return cls(platform=Platform.LOCAL) - - # Platform names are case-insensitive; chat/thread ids keep original case. - # Unknown platforms are treated as local. - if ":" in target_stripped: - parts = target_stripped.split(":", 2) - chat_id = parts[1] if len(parts) > 1 else None - thread_id = parts[2] if len(parts) > 2 else None - try: - platform = Platform(parts[0].lower()) - except ValueError: - return cls(platform=Platform.LOCAL) - return cls(platform=platform, chat_id=chat_id, thread_id=thread_id, is_explicit=True) - + target = target.strip() + if target.lower() == "origin": + return (cls(platform=origin.platform, chat_id=origin.chat_id, thread_id=origin.thread_id, is_origin=True) + if origin else cls(platform=Platform.LOCAL, is_origin=True)) + # Platform names are case-insensitive; chat/thread ids keep case. Unknown platforms -> local. + parts = target.split(":", 2) try: - return cls(platform=Platform(target_lower)) + platform = Platform(parts[0].lower()) except ValueError: return cls(platform=Platform.LOCAL) + return (cls(platform=platform, chat_id=parts[1], thread_id=parts[2] if len(parts) > 2 else None, is_explicit=True) + if len(parts) > 1 else cls(platform=platform)) def to_string(self) -> str: """Convert back to string format.""" @@ -231,23 +129,16 @@ class DeliveryTarget: return "origin" if self.platform == Platform.LOCAL: return "local" - if self.chat_id and self.thread_id: - return f"{self.platform.value}:{self.chat_id}:{self.thread_id}" - if self.chat_id: - return f"{self.platform.value}:{self.chat_id}" - return self.platform.value + parts = [self.platform.value, self.chat_id, self.thread_id if self.chat_id else None] + return ":".join(p for p in parts if p) async def _ensure_named_dm_topic(adapter: Any, chat_id: str, name: str, *, refresh: bool) -> str: """Create (or force-recreate) a named Telegram private DM topic; return its thread id.""" - verb = "refresh" if refresh else "create" - ensure_dm_topic = getattr(adapter, "ensure_dm_topic", None) + verb, ensure_dm_topic = "refresh" if refresh else "create", getattr(adapter, "ensure_dm_topic", None) if ensure_dm_topic is None: raise RuntimeError(f"Telegram adapter cannot {verb} named private DM topics") - if refresh: - thread_id = await ensure_dm_topic(chat_id, name, force_create=True) - else: - thread_id = await ensure_dm_topic(chat_id, name) + thread_id = await ensure_dm_topic(chat_id, name, **({"force_create": True} if refresh else {})) if not thread_id: raise RuntimeError(f"Failed to {verb} Telegram private DM topic '{name}'") return str(thread_id) @@ -257,251 +148,154 @@ class DeliveryRouter: """Resolves delivery targets and dispatches messages to platform adapters.""" def __init__(self, config: GatewayConfig, adapters: Dict[Platform, Any] = None, - dead_targets: Optional[DeadTargetRegistry] = None): - """``dead_targets``: shared registry of confirmed-unreachable targets; - a profile-local registry is created when omitted.""" + dead_targets: Optional[DeadTargetRegistry] = None): # profile-local registry when omitted self.config = config self.adapters = adapters or {} self.output_dir = get_hermes_home() / "cron" / "output" self.dead_targets = dead_targets or DeadTargetRegistry() - async def deliver( - self, - content: str, - targets: List[DeliveryTarget], - job_id: Optional[str] = None, - job_name: Optional[str] = None, - metadata: Optional[Dict[str, Any]] = None - ) -> Dict[str, Any]: + async def deliver(self, content: str, targets: List[DeliveryTarget], job_id: Optional[str] = None, + job_name: Optional[str] = None, metadata: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: """Deliver content to all targets; returns per-target results keyed by target string.""" results = {} - for target in targets: - # Skip targets proven permanently unreachable (deleted group, blocked - # bot, deactivated user) — re-sending each tick wastes flood-control - # budget. Self-healing: a later successful send clears the flag. - # LOCAL/origin-without-chat targets are never dead-tracked. + # Skip targets proven permanently unreachable (deleted group, blocked bot, deactivated user) — + # re-sending each tick wastes flood-control budget. Self-healing: a later successful send + # clears the flag. LOCAL/origin-without-chat targets are never dead-tracked. tracked = target.platform != Platform.LOCAL and target.chat_id if tracked and self.dead_targets.is_dead(target.platform.value, target.chat_id): - logger.info( - "Skipping delivery to known-dead target %s:%s " - "(send to it again to clear)", - target.platform.value, target.chat_id, - ) - results[target.to_string()] = { - "success": False, - "skipped": "dead_target", - "error": "target previously confirmed unreachable", - } + logger.info("Skipping delivery to known-dead target %s:%s (send to it again to clear)", + target.platform.value, target.chat_id) + results[target.to_string()] = {"success": False, "skipped": "dead_target", + "error": "target previously confirmed unreachable"} continue try: if target.platform == Platform.LOCAL: result = self._deliver_local(content, job_id, job_name, metadata) else: result = await self._deliver_to_platform(target, content, metadata) - if target.chat_id and not _send_result_failed(result): + if target.chat_id and _send_result_error(result) is None: self.dead_targets.clear(target.platform.value, target.chat_id) results[target.to_string()] = {"success": True, "result": result} except Exception as e: - # Hard failures raise. Record a whole-chat death so future - # deliveries short-circuit. - if tracked: - dead_kind = _classify_dead_from_error_text(str(e)) - if dead_kind: - self.dead_targets.mark_dead( - target.platform.value, target.chat_id, - reason=f"{dead_kind}: {str(e)[:120]}", - ) + # Hard failures raise. Record a whole-chat death so future deliveries short-circuit. + dead_kind = classify_dead_error(str(e)) if tracked else None + if dead_kind: + self.dead_targets.mark_dead(target.platform.value, target.chat_id, + reason=f"{dead_kind}: {str(e)[:120]}") results[target.to_string()] = {"success": False, "error": str(e)} - return results - def _deliver_local( - self, - content: str, - job_id: Optional[str], - job_name: Optional[str], - metadata: Optional[Dict[str, Any]] - ) -> Dict[str, Any]: + def _deliver_local(self, content: str, job_id: Optional[str], job_name: Optional[str], + metadata: Optional[Dict[str, Any]]) -> Dict[str, Any]: """Save content to local files.""" timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") output_path = self.output_dir / (job_id or "misc") / f"{timestamp}.md" output_path.parent.mkdir(parents=True, exist_ok=True) - - lines = [ - f"# {job_name}" if job_name else "# Delivery Output", - "", - f"**Timestamp:** {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}", - ] - if job_id: - lines.append(f"**Job ID:** {job_id}") - for key, value in (metadata or {}).items(): - lines.append(f"**{key}:** {value}") - lines += ["", "---", "", content] - + lines = [f"# {job_name}" if job_name else "# Delivery Output", "", + f"**Timestamp:** {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}"] + lines += [f"**Job ID:** {job_id}"] if job_id else [] + lines += [f"**{key}:** {value}" for key, value in (metadata or {}).items()] + ["", "---", "", content] output_path.write_text("\n".join(lines), encoding="utf-8") return {"path": str(output_path), "timestamp": timestamp} def _save_full_output(self, content: str, job_id: str) -> Path: """Save full cron output to disk and return the file path.""" timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") - out_dir = get_hermes_home() / "cron" / "output" - out_dir.mkdir(parents=True, exist_ok=True) - path = out_dir / f"{job_id}_{timestamp}.txt" + path = get_hermes_home() / "cron" / "output" / f"{job_id}_{timestamp}.txt" + path.parent.mkdir(parents=True, exist_ok=True) path.write_text(content, encoding="utf-8") return path def _filter_silence_narration_enabled(self) -> bool: - """``HERMES_FILTER_SILENCE_NARRATION`` env overrides the - ``gateway.filter_silence_narration`` config flag (default True).""" + """``HERMES_FILTER_SILENCE_NARRATION`` env overrides the ``gateway.filter_silence_narration`` flag.""" env = os.getenv("HERMES_FILTER_SILENCE_NARRATION") - if env is not None: - return env.strip().lower() in ("1", "true", "yes", "on") - return bool(getattr(self.config, "filter_silence_narration", True)) + return (bool(getattr(self.config, "filter_silence_narration", True)) if env is None + else env.strip().lower() in ("1", "true", "yes", "on")) def _cap_oversized_output(self, adapter: Any, content: str, job_id: str) -> str: - """Audit-save oversized cron output; truncate it for non-chunking adapters. - - Two independent decisions: (1) above MAX_PLATFORM_OUTPUT the full output - is always written to disk as an audit trail, best-effort — a failed save - (full disk, permissions) never blocks delivery; (2) non-chunking adapters - get the content truncated with a footer pointing to the saved file, while - ``splits_long_messages`` adapters receive the full payload. - """ + """Audit-save oversized cron output; truncate it for non-chunking adapters. Above MAX_PLATFORM_OUTPUT + the full output is always written to disk as an audit trail, best-effort — a failed save (full disk, + permissions) never blocks delivery. Non-chunking adapters then get the content truncated with a + footer pointing to the saved file; ``splits_long_messages`` adapters receive the full payload.""" if len(content) <= MAX_PLATFORM_OUTPUT: return content saved_path: Optional[Path] = None try: saved_path = self._save_full_output(content, job_id) except OSError as exc: - logger.warning( - "Audit save failed for cron output (%d chars, job=%s): %s — " - "delivery proceeds without audit copy", - len(content), job_id, exc, - ) - + logger.warning("Audit save failed for cron output (%d chars, job=%s): %s — " + "delivery proceeds without audit copy", len(content), job_id, exc) if getattr(adapter, "splits_long_messages", False): if saved_path: - logger.info( - "Cron output preserved for chunking adapter (%d chars) — " - "full output saved to %s", - len(content), saved_path, - ) + logger.info("Cron output preserved for chunking adapter (%d chars) — " + "full output saved to %s", len(content), saved_path) return content - # The footer needs a valid path: if the best-effort save failed, retry # (a failure now is a real delivery problem and propagates). - if saved_path is None: - saved_path = self._save_full_output(content, job_id) + saved_path = saved_path or self._save_full_output(content, job_id) footer = f"\n\n... [truncated, full output saved to {saved_path}]" - visible = max(0, MAX_PLATFORM_OUTPUT - len(footer)) - logger.info( - "Cron output truncated (%d chars) — full output: %s", - len(content), saved_path, - ) - return content[:visible] + footer + logger.info("Cron output truncated (%d chars) — full output: %s", len(content), saved_path) + return content[:max(0, MAX_PLATFORM_OUTPUT - len(footer))] + footer - async def _deliver_to_platform( - self, - target: DeliveryTarget, - content: str, - metadata: Optional[Dict[str, Any]] - ) -> Dict[str, Any]: + async def _deliver_to_platform(self, target: DeliveryTarget, content: str, + metadata: Optional[Dict[str, Any]]) -> Dict[str, Any]: """Deliver content to a messaging platform.""" transport = resolve_delivery_transport(target.platform, self.config, self.adapters) if transport is None: raise ValueError(f"No adapter configured for {target.platform.value}") - adapter = transport.adapter - if not target.chat_id: raise ValueError(f"No chat ID for {target.platform.value} delivery") + adapter = transport.adapter + content = self._cap_oversized_output(adapter, content, (metadata or {}).get("job_id", "unknown")) - content = self._cap_oversized_output( - adapter, content, (metadata or {}).get("job_id", "unknown") - ) - - # Substrate-level anti-loop guard: drop hallucinated "silence narration" - # (*(silent)*, 🔇, a bare ".") before it reaches any adapter — in - # bot-to-bot channels these mirror back and forth until a model crashes - # with "no content after all retries"; prompt rules drift across - # providers, so this single chokepoint covers every platform. Local/file - # delivery is never filtered (saved silence has no loop risk). - # Cron output is an ARTIFACT, not model chatter: a legitimately terse job - # ("...", a single 🔇) has no mirror loop, and dropping it while returning - # success is how a cron gets logged as delivered with nothing on the - # wire. Cron sends carry job_id in metadata; everything else is filtered. + # Substrate-level anti-loop guard: drop hallucinated "silence narration" (*(silent)*, 🔇, a bare ".") + # before it reaches any adapter — in bot-to-bot channels these mirror back and forth until a model + # crashes with "no content after all retries"; prompt rules drift across providers, so this single + # chokepoint covers every platform. Local/file delivery is never filtered (saved silence has no loop + # risk). Cron output is an ARTIFACT, not model chatter: a legitimately terse job ("...", a single 🔇) + # has no mirror loop, and dropping it while returning success is how a cron gets logged as delivered + # with nothing on the wire. Cron sends carry job_id in metadata; everything else is filtered. is_cron_artifact = "job_id" in (metadata or {}) - if ( - self._filter_silence_narration_enabled() - and not is_cron_artifact - and _is_silence_narration(content) - ): - logger.warning( - "Dropped silence-narration outbound to %s (chat=%s): %r", - target.platform.value, - target.chat_id, - content[:40], - ) - return { - "success": True, - "filtered": "silence_narration", - "delivered": False, - } + if self._filter_silence_narration_enabled() and not is_cron_artifact and _is_silence_narration(content): + logger.warning("Dropped silence-narration outbound to %s (chat=%s): %r", + target.platform.value, target.chat_id, content[:40]) + return {"success": True, "filtered": "silence_narration", "delivered": False} send_metadata = dict(metadata or {}) - if transport.is_relay: - home = self.config.get_home_channel(target.platform) - if home is not None and home.chat_id == target.chat_id: - if home.user_id: - send_metadata["user_id"] = home.user_id - if home.scope_id: - send_metadata["scope_id"] = home.scope_id + home = self.config.get_home_channel(target.platform) if transport.is_relay else None + if home is not None and home.chat_id == target.chat_id: + send_metadata.update({k: v for k, v in (("user_id", home.user_id), ("scope_id", home.scope_id)) if v}) # Caller-supplied thread routing always wins over target.thread_id. - named_telegram_private_topic_name: Optional[str] = None + named_topic: Optional[str] = None # named Telegram private topic created for this send thread_id = target.thread_id - thread_unrouted = thread_id and not any( - key in send_metadata - for key in ("thread_id", "message_thread_id", - "direct_messages_topic_id", "telegram_direct_messages_topic_id") - ) - if thread_unrouted: - is_telegram_private = ( - target.platform == Platform.TELEGRAM - and looks_like_telegram_private_chat_id(target.chat_id) - ) - if is_telegram_private and not _looks_like_int(thread_id): - # Named topic: create via createForumTopic, use message_thread_id directly. - named_telegram_private_topic_name = thread_id - send_metadata["thread_id"] = await _ensure_named_dm_topic( - adapter, target.chat_id, thread_id, refresh=False - ) - send_metadata["telegram_dm_topic_created_for_send"] = True - elif is_telegram_private: - # Legacy numeric private topic ids not created by this send path - # need a reply anchor to stay visible in the requested lane. - if send_metadata.get("telegram_reply_to_message_id") is None: + if thread_id and not any(key in send_metadata for key in _THREAD_ROUTING_KEYS): + send_metadata["thread_id"] = thread_id + if target.platform == Platform.TELEGRAM and looks_like_telegram_private_chat_id(target.chat_id): + if not _looks_like_int(thread_id): + # Named topic: create via createForumTopic, use message_thread_id directly. + named_topic = thread_id + send_metadata["thread_id"] = await _ensure_named_dm_topic(adapter, target.chat_id, thread_id, refresh=False) + send_metadata["telegram_dm_topic_created_for_send"] = True + elif send_metadata.get("telegram_reply_to_message_id") is None: + # Legacy numeric private topic ids not created by this send path need a reply + # anchor to stay visible in the requested lane. raise RuntimeError( "Telegram private DM topic delivery requires telegram_reply_to_message_id; " "send to the bare chat or provide a reply anchor" ) - send_metadata["thread_id"] = thread_id - send_metadata["telegram_dm_topic_reply_fallback"] = True - else: - send_metadata["thread_id"] = thread_id + else: + send_metadata["telegram_dm_topic_reply_fallback"] = True - result = await transport.send( - target.platform, target.chat_id, content, metadata=send_metadata or None, - ) - if _send_result_failed(result): - if named_telegram_private_topic_name and _is_thread_not_found_delivery_error(result): - send_metadata["thread_id"] = await _ensure_named_dm_topic( - adapter, target.chat_id, named_telegram_private_topic_name, refresh=True - ) - send_metadata["telegram_dm_topic_created_for_send"] = True - result = await transport.send( - target.platform, target.chat_id, content, metadata=send_metadata or None, - ) - if _send_result_failed(result): - raise RuntimeError(_send_result_error(result) or f"{target.platform.value} delivery failed") + for retry in (False, True): + result = await transport.send(target.platform, target.chat_id, content, metadata=send_metadata or None) + error = _send_result_error(result) + if retry or error is None or not named_topic or "thread not found" not in error.lower(): + break + # The named topic vanished under us: recreate it once and resend. + send_metadata["thread_id"] = await _ensure_named_dm_topic(adapter, target.chat_id, named_topic, refresh=True) + send_metadata["telegram_dm_topic_created_for_send"] = True + if error is not None: + raise RuntimeError(error or f"{target.platform.value} delivery failed") return result diff --git a/gateway/delivery_ledger.py b/gateway/delivery_ledger.py index 7d5fa26d64..fac8e2b219 100644 --- a/gateway/delivery_ledger.py +++ b/gateway/delivery_ledger.py @@ -1,37 +1,13 @@ -"""Durable delivery-obligation ledger for gateway final responses. - -A final agent response generated but not yet confirmed-delivered is the one -artifact the gateway can lose without a trace: the turn already burned its -tokens, the text exists only in a Python local, and a crash / restart between -finalize and platform ACK drops it silently. - -Each outbound final response gets a small durable row in the shared ``state.db`` -(same conventions as ``tools.async_delegation`` — WAL, owner pid + process-start -liveness, bounded retention). The gateway writes three checkpoints: - - record_obligation() state='pending' before any send attempt - mark_attempting() state='attempting' immediately before the await - mark_delivered() / state='delivered' only on SendResult.success - mark_failed() state='failed' on a definitive rejection - -On startup ``sweep_recoverable()`` claims rows whose owner is dead and hands -them back for redelivery. After an adapter reconnects without a restart, -``sweep_failed_for_runtime()`` may claim only the same live process's -allowlisted transient failures. Crash semantics are explicit about ambiguity -(never silently resend an ambiguous send): - -- ``pending`` — send never started: redeliver plainly, no dup risk. -- ``attempting`` — crashed mid-await: the platform MAY already have it. - Redelivered WITH a visible recovered-reply marker (honest at-least-once). -- ``failed`` — definitively rejected once; restart is a natural retry - boundary. Also carries the marker. -- ``delivered`` — nothing to do; retention prunes. - -Poison rows cannot spin: attempts are capped and stale rows expire, both -transitioning to ``abandoned`` (kept briefly, then pruned). - -Everything here is best-effort: ledger failures must never block or delay an -actual send. Callers wrap every call in try/except. +"""Durable delivery-obligation ledger for gateway final responses (rows in the shared ``state.db``; +WAL, owner pid + process-start liveness, bounded retention) so a crash between finalize and +platform ACK cannot lose a response silently. Checkpoints: record_obligation() 'pending' before +any send | mark_attempting() 'attempting' right before the await | mark_delivered() 'delivered' +only on SendResult.success | mark_failed() 'failed' on a definitive rejection. Crash semantics +(never silently resend an ambiguous send): pending = never started, redeliver plainly; attempting += crashed mid-await, platform MAY have it, redeliver WITH a visible recovered marker; failed = +rejected once, restart is a retry boundary, also marked; delivered = prune. Attempts are capped +and stale rows expire, both -> 'abandoned' (kept briefly, then pruned). Everything is +best-effort: ledger failures must never block a send; callers wrap every call in try/except. """ from __future__ import annotations @@ -42,13 +18,12 @@ import os import sqlite3 import threading import time -from contextlib import contextmanager +from contextlib import closing, contextmanager from typing import Any, Dict, Iterator, List, Optional from hermes_constants import get_hermes_home logger = logging.getLogger(__name__) - _DB_LOCK = threading.Lock() # Redelivery policy knobs (deliberately not config — the ledger is gated by @@ -58,23 +33,15 @@ STALE_AFTER_SECONDS = 24 * 60 * 60 _RETENTION_SECONDS = 7 * 24 * 60 * 60 _MAX_ROWS = 500 -# Visible prefix for redeliveries that might duplicate an already-received -# message (crash mid-send / post-rejection retry). Honest at-least-once. -RECOVERED_MARKER = ( - "♻️ Recovered reply — the gateway restarted during delivery, " - "so this may be a duplicate:\n\n" -) +# Visible prefixes for redeliveries that might duplicate an already-received message (crash mid-send / +# post-rejection retry) — honest at-least-once. Runtime recovery uses a distinct marker: no restart +# occurred, but a network rejection's acknowledgement can still have been lost independently. +RECOVERED_MARKER = "♻️ Recovered reply — the gateway restarted during delivery, so this may be a duplicate:\n\n" +RECONNECTED_MARKER = ("♻️ Recovered reply — the messaging platform reconnected after the original " + "delivery failed, so this may be a duplicate:\n\n") -# Runtime recovery uses a distinct marker: no restart occurred, but a network -# rejection's acknowledgement can still have been lost independently. -RECONNECTED_MARKER = ( - "♻️ Recovered reply — the messaging platform reconnected after the original " - "delivery failed, so this may be a duplicate:\n\n" -) - -# Runtime replay is fail-closed: only errors whose send contract proves they are -# transient reconnect failures. Permanent rejects (blocked bot, bad auth, missing -# chat) must not be retried merely because an adapter reconnected. +# Runtime replay is fail-closed: only errors whose send contract proves they are transient reconnect +# failures. Permanent rejects (blocked bot, bad auth, missing chat) must not be retried on reconnect. _RUNTIME_RETRYABLE_ERRORS = frozenset({"send_path_degraded"}) @@ -89,15 +56,13 @@ def _connect() -> sqlite3.Connection: try: _initialize_schema(conn) except Exception: - # A PRAGMA/DDL failure after connect() must not leak the connection. - conn.close() + conn.close() # a PRAGMA/DDL failure after connect() must not leak the connection raise return conn def _initialize_schema(conn: sqlite3.Connection) -> None: from hermes_state import apply_wal_with_fallback - apply_wal_with_fallback(conn, db_label="state.db (delivery_ledger)") conn.execute( """CREATE TABLE IF NOT EXISTS delivery_obligations ( @@ -117,8 +82,7 @@ def _initialize_schema(conn: sqlite3.Connection) -> None: adapter_profile TEXT )""" ) - columns = {row[1] for row in conn.execute("PRAGMA table_info(delivery_obligations)")} - if "adapter_profile" not in columns: + if "adapter_profile" not in {row[1] for row in conn.execute("PRAGMA table_info(delivery_obligations)")}: try: conn.execute("ALTER TABLE delivery_obligations ADD COLUMN adapter_profile TEXT") except sqlite3.OperationalError as exc: @@ -129,29 +93,25 @@ def _initialize_schema(conn: sqlite3.Connection) -> None: @contextmanager def _transaction() -> Iterator[sqlite3.Connection]: - """Open a connection, commit/rollback on exit, and ALWAYS close it. - - ``sqlite3.Connection`` as a context manager only commits/rolls back; it does - not close. ``with _connect()`` alone would leak a connection (and its WAL/SHM - fds) per call until GC — ``record_obligation`` runs on every outbound final - response, so this ledger would exhaust ``RLIMIT_NOFILE`` on a long-running gateway. - """ + """Open a connection, commit/rollback on exit, and ALWAYS close it: ``sqlite3.Connection`` as a + context manager only commits/rolls back, so ``with _connect()`` alone leaks a connection (and its + WAL/SHM fds) per call — ``record_obligation`` runs on every final response; exhausts RLIMIT_NOFILE.""" conn = _connect() + with closing(conn), conn: + yield conn + + +def _start_time(pid: int) -> Optional[int]: try: - with conn: - yield conn - finally: - conn.close() + from gateway.status import get_process_start_time # lazy: tests monkeypatch gateway.status + return get_process_start_time(pid) + except Exception: + return None def _owner_stamp() -> tuple[int, Optional[int]]: pid = os.getpid() - try: - from gateway.status import get_process_start_time - - return pid, get_process_start_time(pid) - except Exception: - return pid, None + return pid, _start_time(pid) def _owner_alive(pid: Any, started_at: Any) -> bool: @@ -162,18 +122,11 @@ def _owner_alive(pid: Any, started_at: Any) -> bool: pid = int(pid) except (TypeError, ValueError): return False - try: - from gateway.status import get_process_start_time - - current_start = get_process_start_time(pid) - except Exception: - current_start = None + current_start = _start_time(pid) if current_start is None: - # Start time unreadable: alive iff the pid exists. Route through the - # cross-platform probe — ``os.kill(pid, 0)`` on Windows is NOT a no-op - # (bpo-14484: it maps to ``GenerateConsoleCtrlEvent(0, pid)`` and could - # Ctrl+C the gateway's own console group). ``_pid_exists`` keeps the - # EPERM-means-alive semantics (pid exists but is owned by another user). + # Start time unreadable: alive iff the pid exists. Route through the cross-platform probe — on Windows + # ``os.kill(pid, 0)`` is NOT a no-op (bpo-14484: it maps to ``GenerateConsoleCtrlEvent(0, pid)`` and could + # Ctrl+C the gateway's own console group). ``_pid_exists`` keeps EPERM-means-alive (pid owned by another user). try: from gateway.status import _pid_exists except Exception: @@ -181,47 +134,29 @@ def _owner_alive(pid: Any, started_at: Any) -> bool: return False # never fall back to a raw sig-0 probe on Windows try: os.kill(pid, 0) # windows-footgun: ok — POSIX-only fallback branch - except ProcessLookupError: - return False - except PermissionError: return True - except OSError: - return False - return True + except OSError as exc: # incl. ProcessLookupError; EPERM means the pid exists + return isinstance(exc, PermissionError) try: return bool(_pid_exists(pid)) except Exception: return False - if started_at is None: - return True try: - return int(current_start) == int(started_at) + return started_at is None or int(current_start) == int(started_at) except (TypeError, ValueError): return True def compute_obligation_id(session_key: str, message_ref: str, content: str) -> str: - """Stable id: same turn + same content re-records idempotently, while distinct - threads/topics on one chat never collide (session_key carries platform, chat and - thread; ``message_ref`` is the triggering inbound message id).""" - payload = f"{session_key}|{message_ref}|{content}" - return hashlib.sha256(payload.encode("utf-8", "replace")).hexdigest()[:24] + """Stable id: same turn + same content re-records idempotently, while distinct threads/topics on one + chat never collide (session_key carries platform/chat/thread; ``message_ref`` = inbound message id).""" + return hashlib.sha256(f"{session_key}|{message_ref}|{content}".encode("utf-8", "replace")).hexdigest()[:24] -def record_obligation( - *, - obligation_id: str, - session_key: str, - platform: str, - chat_id: str, - thread_id: Optional[str], - content: str, - adapter_profile: Optional[str] = None, -) -> None: +def record_obligation(*, obligation_id: str, session_key: str, platform: str, chat_id: str, + thread_id: Optional[str], content: str, adapter_profile: Optional[str] = None) -> None: """Record a final response as owed to the platform (state='pending').""" - now = time.time() - stored_profile = str(adapter_profile).strip() if adapter_profile else "default" - pid, started = _owner_stamp() + now, (pid, started) = time.time(), _owner_stamp() with _DB_LOCK, _transaction() as conn: conn.execute( """INSERT OR REPLACE INTO delivery_obligations @@ -229,10 +164,8 @@ def record_obligation( content, state, attempts, created_at, updated_at, owner_pid, owner_started_at, adapter_profile) VALUES (?, ?, ?, ?, ?, ?, 'pending', 0, ?, ?, ?, ?, ?)""", - (obligation_id, session_key, platform, str(chat_id), - str(thread_id) if thread_id else None, content, now, now, - pid, started, stored_profile), - ) + (obligation_id, session_key, platform, str(chat_id), str(thread_id) if thread_id else None, + content, now, now, pid, started, str(adapter_profile).strip() if adapter_profile else "default")) _prune() @@ -251,11 +184,9 @@ def mark_failed(obligation_id: str, error: str = "") -> None: def release_runtime_claim(obligation_id: str, error: str = "") -> bool: """Return an unsent runtime claim to ``failed`` without spending an attempt. - Runtime recovery claims before clearing ``resume_pending`` so two reconnect - paths cannot send the same row. If the session flag cannot be cleared, no - send was attempted and the claim must not consume the redelivery budget. - Fail-closed to the exact current process instance and ``attempting`` state. - """ + Runtime recovery claims before clearing ``resume_pending`` so two reconnect paths cannot send the + same row; if the flag cannot be cleared no send was attempted and the claim must not consume the + redelivery budget. Fail-closed to the exact current process instance and ``attempting`` state.""" pid, started = _owner_stamp() if started is None: return False @@ -267,9 +198,7 @@ def release_runtime_claim(obligation_id: str, error: str = "") -> bool: updated_at=?, last_error=? WHERE obligation_id=? AND state='attempting' AND owner_pid IS ? AND owner_started_at IS ?""", - (time.time(), error[:500] if error else None, - obligation_id, pid, started), - ) + (time.time(), error[:500] if error else None, obligation_id, pid, started)) return bool(cursor.rowcount) @@ -279,39 +208,31 @@ def _update_state(obligation_id: str, state: str, error: str = "") -> None: """UPDATE delivery_obligations SET state=?, updated_at=?, last_error=? WHERE obligation_id=?""", - (state, time.time(), error[:500] if error else None, obligation_id), - ) + (state, time.time(), error[:500] if error else None, obligation_id)) -def _exhausted(attempts: int, created_at: float, now: float) -> bool: - return attempts >= MAX_ATTEMPTS or (now - created_at) > STALE_AFTER_SECONDS +def _claimed_row(oid, session_key, platform, chat_id, thread_id, content, attempts, profile, *, + needs_marker: bool, runtime: bool = False) -> Dict[str, Any]: + """Claimed-row dict handed back for redelivery; ``runtime`` adds the reconnect-marker fields.""" + return {"obligation_id": oid, "session_key": session_key, "platform": platform, "chat_id": chat_id, + "thread_id": thread_id, "content": content, "needs_marker": needs_marker, + **({"marker": RECONNECTED_MARKER} if runtime else {}), "profile": profile, + **({"runtime_recovery": True} if runtime else {}), "attempts": attempts + 1} -def sweep_recoverable( - now: Optional[float] = None, - *, - deliverable_platforms: Optional[set] = None, - deliverable_targets: Optional[set] = None, -) -> List[Dict[str, Any]]: +def sweep_recoverable(now: Optional[float] = None, *, deliverable_platforms: Optional[set] = None, + deliverable_targets: Optional[set] = None) -> List[Dict[str, Any]]: """Claim undelivered rows owned by dead processes; return them for redelivery. - Claiming atomically re-stamps the owner to THIS process and increments - ``attempts`` (the UPDATE is guarded on the previous owner stamp, so a second - gateway racing the same sweep cannot double-claim). Rows over the attempts - cap or stale cutoff transition to 'abandoned' instead of being returned. - - ``deliverable_platforms`` restricts claiming to platforms the caller can - actually send on this boot: ``attempts`` is the redelivery budget and must - only be spent on a real send, else a platform that failed to connect burns - one attempt per boot and hits the cap having never been sent once. Rows for - absent platforms are left untouched (the stale cutoff still bounds them). - - ``deliverable_targets`` further scopes multiplexed gateways by exact - ``(platform, adapter_profile)`` so one connected bot cannot spend another - disconnected bot's retry budget. - """ - now = now if now is not None else time.time() - pid, started = _owner_stamp() + Claiming atomically re-stamps the owner to THIS process and increments ``attempts`` (the UPDATE is + guarded on the previous owner stamp, so a second gateway racing the same sweep cannot double-claim). + Rows over the attempts cap or stale cutoff become 'abandoned'. ``deliverable_platforms`` restricts + claiming to platforms the caller can send on this boot: ``attempts`` is the redelivery budget and + must only be spent on a real send, else a platform that failed to connect burns one attempt per boot + and hits the cap having never been sent once (the stale cutoff still bounds untouched rows). + ``deliverable_targets`` further scopes multiplexed gateways by exact ``(platform, adapter_profile)`` + so one connected bot cannot spend another disconnected bot's retry budget.""" + now, (pid, started) = now if now is not None else time.time(), _owner_stamp() claimed: List[Dict[str, Any]] = [] with _DB_LOCK, _transaction() as conn: rows = conn.execute( @@ -321,73 +242,45 @@ def sweep_recoverable( FROM delivery_obligations WHERE state IN ('pending', 'attempting', 'failed')""" ).fetchall() - for (oid, session_key, platform, chat_id, thread_id, content, state, - attempts, created_at, owner_pid, owner_started_at, - adapter_profile) in rows: + for (oid, session_key, platform, chat_id, thread_id, content, state, attempts, created_at, + owner_pid, owner_started_at, adapter_profile) in rows: if _owner_alive(owner_pid, owner_started_at): continue # a live gateway still owns this row - if _exhausted(attempts, created_at, now): + if attempts >= MAX_ATTEMPTS or (now - created_at) > STALE_AFTER_SECONDS: # exhausted -> abandoned conn.execute( """UPDATE delivery_obligations - SET state='abandoned', updated_at=? WHERE obligation_id=?""", - (now, oid), - ) + SET state='abandoned', updated_at=? WHERE obligation_id=?""", (now, oid)) continue - if deliverable_platforms is not None and platform not in deliverable_platforms: + if ((deliverable_platforms is not None and platform not in deliverable_platforms) + or (deliverable_targets is not None and (platform, adapter_profile) not in deliverable_targets)): continue # no adapter this boot — claiming would spend an attempt on a no-op - if deliverable_targets is not None and (platform, adapter_profile) not in deliverable_targets: - continue cursor = conn.execute( """UPDATE delivery_obligations SET owner_pid=?, owner_started_at=?, attempts=attempts+1, updated_at=? WHERE obligation_id=? AND (owner_pid IS ? OR owner_pid=?)""", - (pid, started, now, oid, owner_pid, owner_pid), - ) - if cursor.rowcount: - claimed.append({ - "obligation_id": oid, - "session_key": session_key, - "platform": platform, - "chat_id": chat_id, - "thread_id": thread_id, - "content": content, - # pending = never started, redeliver plainly; else carry marker. - "needs_marker": state != "pending", - "profile": adapter_profile, - "attempts": attempts + 1, - }) + (pid, started, now, oid, owner_pid, owner_pid)) + if cursor.rowcount: # pending = never started, redeliver plainly; else carry marker + claimed.append(_claimed_row(oid, session_key, platform, chat_id, thread_id, content, attempts, + adapter_profile, needs_marker=state != "pending")) return claimed -def sweep_failed_for_runtime( - platform: str, - now: Optional[float] = None, - *, - profile: Optional[str] = None, -) -> List[Dict[str, Any]]: +def sweep_failed_for_runtime(platform: str, now: Optional[float] = None, *, + profile: Optional[str] = None) -> List[Dict[str, Any]]: """Claim this process's reconnect-retryable failed rows for one adapter. - ``profile`` scopes multiplexed gateways to the bot identity that owned the - failed send; ``None`` means the primary/default adapter. The persisted - adapter owner is independent of the routed session namespace. Unowned rows - and rows owned by another process are left for the startup/dead-owner sweep. - - Startup recovery ignores rows owned by a live gateway, so a final response - rejected with ``send_path_degraded`` would stay stranded when only the - adapter reconnects. This sweep closes that gap without weakening ownership: - only rows stamped to this exact process instance, only allowlisted transient - errors, same attempts/staleness bounds, every update guarded by the prior - owner stamp and ``failed`` state. Claimed rows always carry the reconnect - marker because the failed send's acknowledgement is not safe to infer. - """ - now = now if now is not None else time.time() - pid, started = _owner_stamp() - if started is None: - # PID alone cannot distinguish this process from a stale row left after - # PID reuse. Runtime replay is optional, so fail closed; startup - # recovery remains the durable fallback. - return [] + ``profile`` scopes multiplexed gateways to the bot identity that owned the failed send (``None`` = + primary/default adapter); unowned rows and rows owned by another process are left for the + startup/dead-owner sweep. Startup recovery ignores rows owned by a live gateway, so a response + rejected with ``send_path_degraded`` would stay stranded when only the adapter reconnects; this closes + that gap without weakening ownership: only rows stamped to this exact process instance, only + allowlisted transient errors, same attempts/staleness bounds, every update guarded by the prior owner + stamp and ``failed`` state. Claimed rows always carry the reconnect marker (the failed send's ack is + not safe to infer).""" + now, (pid, started) = now if now is not None else time.time(), _owner_stamp() + if started is None: # PID alone cannot distinguish this process from a stale row left after PID + return [] # reuse; runtime replay is optional, so fail closed (startup recovery remains). expected_profile = "default" if not profile or profile == "default" else str(profile) claimed: List[Dict[str, Any]] = [] with _DB_LOCK, _transaction() as conn: @@ -396,66 +289,41 @@ def sweep_failed_for_runtime( content, attempts, created_at, owner_pid, owner_started_at, last_error, adapter_profile FROM delivery_obligations - WHERE state='failed' AND platform=?""", - (platform,), - ).fetchall() - for (oid, session_key, row_platform, chat_id, thread_id, content, - attempts, created_at, owner_pid, owner_started_at, last_error, - adapter_profile) in rows: - if adapter_profile != expected_profile: - continue + WHERE state='failed' AND platform=?""", (platform,)).fetchall() + for (oid, session_key, row_platform, chat_id, thread_id, content, attempts, created_at, + owner_pid, owner_started_at, last_error, adapter_profile) in rows: # Exact process-start matching prevents PID reuse from stealing work. - if owner_pid != pid or owner_started_at != started: + if (adapter_profile != expected_profile or owner_pid != pid or owner_started_at != started + or str(last_error or "").strip().lower() not in _RUNTIME_RETRYABLE_ERRORS): continue - if str(last_error or "").strip().lower() not in _RUNTIME_RETRYABLE_ERRORS: - continue - owner_guard = (oid, owner_pid, owner_started_at) - if _exhausted(attempts, created_at, now): + owner_guard = (now, oid, owner_pid, owner_started_at) + if attempts >= MAX_ATTEMPTS or (now - created_at) > STALE_AFTER_SECONDS: # exhausted -> abandoned conn.execute( """UPDATE delivery_obligations SET state='abandoned', updated_at=? WHERE obligation_id=? AND state='failed' - AND owner_pid IS ? AND owner_started_at IS ?""", - (now, *owner_guard), - ) + AND owner_pid IS ? AND owner_started_at IS ?""", owner_guard) continue cursor = conn.execute( """UPDATE delivery_obligations SET state='attempting', attempts=attempts+1, updated_at=? WHERE obligation_id=? AND state='failed' - AND owner_pid IS ? AND owner_started_at IS ?""", - (now, *owner_guard), - ) + AND owner_pid IS ? AND owner_started_at IS ?""", owner_guard) if cursor.rowcount: - claimed.append({ - "obligation_id": oid, - "session_key": session_key, - "platform": row_platform, - "chat_id": chat_id, - "thread_id": thread_id, - "content": content, - "needs_marker": True, - "marker": RECONNECTED_MARKER, - "profile": adapter_profile, - "runtime_recovery": True, - "attempts": attempts + 1, - }) + claimed.append(_claimed_row(oid, session_key, row_platform, chat_id, thread_id, content, + attempts, adapter_profile, needs_marker=True, runtime=True)) return claimed def _prune(now: Optional[float] = None) -> None: now = now if now is not None else time.time() - cutoff = now - _RETENTION_SECONDS try: with _transaction() as conn: conn.execute( """DELETE FROM delivery_obligations - WHERE state IN ('delivered', 'abandoned') AND updated_at < ?""", - (cutoff,), - ) + WHERE state IN ('delivered', 'abandoned') AND updated_at < ?""", (now - _RETENTION_SECONDS,)) total = conn.execute("SELECT COUNT(*) FROM delivery_obligations").fetchone()[0] - excess = max(0, total - _MAX_ROWS) - if excess: + if total > _MAX_ROWS: conn.execute( """DELETE FROM delivery_obligations WHERE obligation_id IN ( SELECT obligation_id FROM delivery_obligations @@ -464,9 +332,7 @@ def _prune(now: Optional[float] = None) -> None: WHEN 'abandoned' THEN 1 ELSE 2 END, updated_at ASC - LIMIT ?)""", - (excess,), - ) + LIMIT ?)""", (total - _MAX_ROWS,)) except Exception: logger.debug("delivery ledger prune failed", exc_info=True) @@ -476,12 +342,8 @@ def ledger_enabled(config: Optional[Dict[str, Any]] = None) -> bool: try: if config is None: from hermes_cli.config import load_config - config = load_config() - gw = config.get("gateway") or {} - value = gw.get("delivery_ledger", True) - if isinstance(value, str): - return value.strip().lower() not in {"false", "0", "no", "off"} - return bool(value) + value = (config.get("gateway") or {}).get("delivery_ledger", True) + return value.strip().lower() not in {"false", "0", "no", "off"} if isinstance(value, str) else bool(value) except Exception: return True diff --git a/gateway/disk_status.py b/gateway/disk_status.py index 22d0102b57..897a251bc4 100644 --- a/gateway/disk_status.py +++ b/gateway/disk_status.py @@ -1,86 +1,49 @@ """Disk-usage rollup for ``/api/status``. Companion to :mod:`gateway.memory_status`: a hosted agent can fill its data -volume (SQLite writes failing, sessions and config saves lost) while its -dashboard and the NAS agent card look healthy. ``gateway/readiness.py`` -already probes disk, but readiness is a component verdict, not user-facing -telemetry — this module produces the public block the dashboard SPA and the -NAS availability sweep consume. - -Disk is sampled live via :func:`shutil.disk_usage` (one ``statvfs`` call), so -there is no staleness dimension and no ``sampled_at``. - -Public-safety: ``/api/status`` is unauthenticated (``PUBLIC_API_PATHS``); this -block carries only coarse numbers (MB, one-decimal percent) and an enum — the same -disclosure class as the ``memory`` block. - -Best-effort and read-only: an unreadable filesystem degrades to +volume (SQLite writes failing, config saves lost) while its dashboard looks +healthy. Sampled live via one ``statvfs`` call, so there is no ``sampled_at``. +``/api/status`` is unauthenticated: only coarse numbers (MB, one-decimal +percent) and an enum. Best-effort: an unreadable filesystem degrades to ``pressure="unknown"`` rather than raising into the status endpoint. """ from __future__ import annotations -import logging import shutil from pathlib import Path from typing import Any, Dict, Optional -logger = logging.getLogger(__name__) - -# Percent alone misleads both ways: 90% used on 100 GB leaves 10 GB, while 50% -# on a tiny volume is one download from write failures. So percent triggers are -# gated on absolute headroom also being low, and a hard absolute floor applies -# regardless of size (below it SQLite journaling / config writes are at risk). -_CRITICAL_FREE_MB = 256 # < 256 MB free: critical on any volume -_CRITICAL_PERCENT = 95.0 # >= 95% used AND < 1 GB free: critical -_CRITICAL_HEADROOM_MB = 1024 -_ELEVATED_FREE_MB = 512 # < 512 MB free: elevated on any volume -_ELEVATED_PERCENT = 85.0 # >= 85% used AND < 4 GB free: elevated -_ELEVATED_HEADROOM_MB = 4096 +from gateway.memory_status import _nonneg_int +# Percent alone misleads both ways: 90% used on 100 GB leaves 10 GB, while 50% on a tiny +# volume is one download from write failures. So percent triggers are gated on absolute +# headroom also being low, and a hard floor applies regardless of size (below it SQLite +# journaling / config writes are at risk). Rows: (level, free floor, used-%, headroom); worst first. +_PRESSURE_TIERS = ( + ("critical", 256, 95.0, 1024), # < 256 MB free, or >= 95% used AND < 1 GB free + ("elevated", 512, 85.0, 4096), # < 512 MB free, or >= 85% used AND < 4 GB free +) _BYTES_PER_MB = 1024 * 1024 -def _coerce_mb(value: Any) -> Optional[int]: - if isinstance(value, bool) or not isinstance(value, int) or value < 0: - return None - return value - - def classify_disk_pressure(free_mb: Any, total_mb: Any) -> str: - """Map free/total MB to ``ok``/``elevated``/``critical``. - - ``unknown`` when the sample is missing or malformed — the caller must - not treat "we could not read it" as "disk is fine". - """ - free = _coerce_mb(free_mb) - total = _coerce_mb(total_mb) - if free is None or total is None or total <= 0: + """``ok``/``elevated``/``critical`` from free/total MB; ``unknown`` when the sample + is missing/malformed — "could not read it" must never read as "fine".""" + free, total = _nonneg_int(free_mb), _nonneg_int(total_mb) + if free is None or not total: return "unknown" used_percent = (1 - free / total) * 100.0 - for level, free_floor, percent_floor, headroom in ( - ("critical", _CRITICAL_FREE_MB, _CRITICAL_PERCENT, _CRITICAL_HEADROOM_MB), - ("elevated", _ELEVATED_FREE_MB, _ELEVATED_PERCENT, _ELEVATED_HEADROOM_MB), - ): + for level, free_floor, percent_floor, headroom in _PRESSURE_TIERS: if free < free_floor or (used_percent >= percent_floor and free < headroom): return level return "ok" def collect_disk_status(home: Optional[Path] = None) -> Dict[str, Any]: - """Build the ``disk`` block for ``/api/status``. - - ``home`` scopes the sample to a profile's HERMES_HOME (same contract as the - ``memory`` block; on hosted images every profile shares one volume). - Always returns a dict and never raises — an unreadable/unmounted filesystem - yields ``{"pressure": "unknown", ...}``. - """ - status: Dict[str, Any] = { - "pressure": "unknown", - "total_mb": None, - "free_mb": None, - "used_percent": None, - } + """``disk`` block for ``/api/status`` (same ``home`` contract as ``memory``). + Never raises — an unreadable/unmounted filesystem yields ``pressure="unknown"``.""" + status: Dict[str, Any] = {"pressure": "unknown", "total_mb": None, "free_mb": None, "used_percent": None} try: if home is None: from hermes_constants import get_hermes_home @@ -91,10 +54,7 @@ def collect_disk_status(home: Optional[Path] = None) -> Dict[str, Any]: return status if usage.total <= 0: return status - total_mb = usage.total // _BYTES_PER_MB - free_mb = usage.free // _BYTES_PER_MB - status["total_mb"] = total_mb - status["free_mb"] = free_mb - status["used_percent"] = round((usage.used / usage.total) * 100, 1) - status["pressure"] = classify_disk_pressure(free_mb, total_mb) + total_mb, free_mb = usage.total // _BYTES_PER_MB, usage.free // _BYTES_PER_MB + status.update(total_mb=total_mb, free_mb=free_mb, used_percent=round((usage.used / usage.total) * 100, 1), + pressure=classify_disk_pressure(free_mb, total_mb)) return status diff --git a/gateway/display_config.py b/gateway/display_config.py index a02a62c137..85ad1fb3e5 100644 --- a/gateway/display_config.py +++ b/gateway/display_config.py @@ -1,63 +1,42 @@ -"""Per-platform display/verbosity configuration resolver. +"""Per-platform display/verbosity resolver (``resolve_display_setting``). -``resolve_display_setting()`` is the single entry-point. Resolution order -(first non-None wins): - 1. ``display.platforms..`` — explicit per-platform override - 2. ``display.`` — global user setting - 3. ``_PLATFORM_DEFAULTS[][]`` — built-in platform default - 4. ``_GLOBAL_DEFAULTS[]`` — built-in global default - -Exception: ``display.streaming`` is CLI-only. Gateway streaming follows the -top-level ``streaming`` config unless ``display.platforms..streaming`` -sets an explicit per-platform override. - -Backward compatibility: ``display.tool_progress_overrides`` is still read as a -fallback for ``tool_progress`` when no ``display.platforms`` entry exists (a -config migration moves the old format into ``display.platforms``). +Resolution order, first non-None wins: ``display.platforms..`` → +``display.`` → ``_PLATFORM_DEFAULTS[platform][key]`` → ``_GLOBAL_DEFAULTS[key]``. +Exception: ``display.streaming`` is CLI-only; gateway streaming follows the top-level +``streaming`` config unless a per-platform override sets it. Legacy +``display.tool_progress_overrides`` is still read as a ``tool_progress`` fallback. """ from __future__ import annotations from typing import Any -# Settings configurable per-platform, with global defaults. Other display -# settings (compact, personality, skin, ...) are CLI-only. +# Settings configurable per-platform; other display settings are CLI-only. _GLOBAL_DEFAULTS: dict[str, Any] = { "tool_progress": "all", "tool_progress_grouping": "accumulate", # "accumulate" = edit one bubble; "separate" = one msg per tool "show_reasoning": False, - # Reasoning summary rendering: "code" (💭 **Reasoning:** + fenced block), - # "blockquote" ("> " lines), "subtext" ("-# " Discord small grey text). + # "code" (💭 **Reasoning:** + fenced block), "blockquote" ("> "), "subtext" ("-# " Discord). "reasoning_style": "code", "tool_preview_length": 0, "streaming": None, # None = follow top-level streaming config - # Gateway-only assistant/status chatter; default on for back-compat, mobile - # platforms opt down to final-answer-first. + # Gateway-only assistant/status chatter; mobile platforms opt down to final-answer-first. "interim_assistant_messages": True, "long_running_notifications": True, "busy_ack_detail": True, - # busy_input_mode=steer confirmation echo ("Steered into current run"). - # Disabling only suppresses the echo; the text still lands in the run. + # busy_input_mode=steer echo ("Steered into current run"); the text still lands in the run. "busy_steer_ack_enabled": True, - # Delete tool-progress / "⏳ Working" / status bubbles after the final - # response on platforms that support deletion (e.g. Telegram). Off by - # default; progress is still shown live, only cleaned up after success so - # the chat doesn't fill with stale breadcrumbs. Failed runs leave bubbles - # in place as breadcrumbs. + # Delete tool-progress / "⏳ Working" bubbles after a SUCCESSFUL final response on + # platforms that support deletion (Telegram); failed runs keep them as breadcrumbs. "cleanup_progress": False, - # Live working-state text on platforms whose typing indicator renders text - # (Slack assistant status): "full"/true = verb + argument preview, - # "verb" = verb only (keeps paths/commands out of shared channels), - # "off"/false = static text. Independent of tool_progress and free: the - # existing typing refresh cadence just renders different text. + # Working-state text on text-rendering indicators (Slack assistant status): + # "full"/true = verb + argument preview, "verb" = verb only (keeps paths out of + # shared channels), "off"/false = static text. "live_status": "full", } -# Per-platform defaults tiered by capability: -# HIGH — message editing, personal/team use -# MEDIUM — editing, but often workspace/customer-facing -# LOW — no edit support: each progress message is permanent -# MINIMAL — batch / non-interactive delivery +# Tiers: HIGH = editing, personal/team use; MEDIUM = editing but customer-facing; +# LOW = no edit support (progress messages are permanent); MINIMAL = batch delivery. _TIER_HIGH = { "tool_progress": "all", "show_reasoning": False, @@ -79,8 +58,7 @@ _TIER_LOW = { _TIER_MINIMAL = {**_TIER_LOW, "tool_preview_length": 0} _PLATFORM_DEFAULTS: dict[str, dict[str, Any]] = { - # Telegram is usually a mobile inbox: quiet tool_progress and no busy-ack - # iteration counter, but DO surface interim assistant commentary and + # Mobile inbox: quiet tool_progress / busy-ack, but keep interim commentary and # heartbeats so it doesn't look like "typing..." for 30 minutes. "telegram": {**_TIER_HIGH, "tool_progress": "off", "busy_ack_detail": False}, # Discord's "-# " subtext reads as metadata, so reasoning defaults to it. @@ -96,25 +74,18 @@ _PLATFORM_DEFAULTS: dict[str, dict[str, Any]] = { "mattermost": _TIER_MEDIUM, "matrix": _TIER_MEDIUM, "feishu": _TIER_MEDIUM, - # Buzz (Nostr via buzz-cli) can edit in place but channels are shared - # community spaces; without an entry it inherited the verbose globals. + # Buzz (Nostr) can edit in place but channels are shared community spaces. "buzz": _TIER_MEDIUM, "signal": _TIER_LOW, "whatsapp": _TIER_MEDIUM, # Baileys bridge supports /edit - # Cloud API supports editing but our adapter lacks edit_message; promote - # to MEDIUM once it lands. - "whatsapp_cloud": _TIER_LOW, - # Photon and BlueBubbles are permanent-message iMessage inboxes (no edit); - # without an entry Photon inherited the noisy "all" globals and narrated - # on nearly every turn. + "whatsapp_cloud": _TIER_LOW, # adapter lacks edit_message; promote once it lands + # Permanent-message iMessage inboxes (no edit). "photon": _TIER_LOW, "bluebubbles": _TIER_LOW, "weixin": _TIER_LOW, - # WeCom is non-editable but has a native streaming transport (msgtype - # "stream") the consumer routes mid-stream content through; streaming on - # gives the client a typing animation + cumulative updates instead of a - # single one-shot markdown drop. + # Non-editable, but its native "stream" msgtype gives a typing animation + + # cumulative updates instead of a one-shot markdown drop. "wecom": {**_TIER_LOW, "streaming": True}, "wecom_callback": _TIER_LOW, "dingtalk": _TIER_LOW, @@ -155,8 +126,7 @@ def resolve_display_setting( if isinstance(legacy, dict) and legacy.get(platform_key) is not None: return _normalise(setting, legacy[platform_key]) - # 2. Global user setting. display.streaming controls only CLI terminal - # streaming; gateway streaming follows the top-level config + platform overrides. + # 2. Global user setting (display.streaming is CLI-only, see module docstring). if setting != "streaming" and display_cfg.get(setting) is not None: return _normalise(setting, display_cfg[setting]) @@ -167,9 +137,7 @@ def resolve_display_setting( return fallback if val is None else val -# --------------------------------------------------------------------------- -# Normalisation of YAML quirks (bare ``off`` → False in YAML 1.1, etc.) -# --------------------------------------------------------------------------- +# --- Normalisation of YAML quirks (bare ``off`` → False in YAML 1.1, etc.) --- _TRUTHY = {"true", "1", "yes", "on"} _FALSY = {"false", "0", "no"} diff --git a/gateway/drain_control.py b/gateway/drain_control.py index e1e323368a..232d45ae4a 100644 --- a/gateway/drain_control.py +++ b/gateway/drain_control.py @@ -1,38 +1,18 @@ """External drain-control marker contract (dashboard → gateway). -There is no control channel into a running gateway, so the dashboard's -begin/cancel-drain endpoint talks to it the same way ``.restart_notify.json`` -does: it writes (or removes) a marker file and a gateway watcher reacts. This -module owns the contract so writer and reader can never disagree. - -Contract (presence-based): - * begin-drain → write ``{HERMES_HOME}/.drain_request.json`` with - ``{"action": "drain", "requested_at": , "principal": , - "epoch": , "suppress_notification": }``. - * cancel-drain → remove the marker. - * The watcher treats presence of an ACTIVE marker as "external drain": flip - ``gateway_state -> "draining"`` and stop accepting new turns. Absence or a - stale marker means "not draining" (revert to ``running``). - -Staleness — two independent, individually-lenient signals (either suffices): - * epoch mismatch: HERMES_HOME is a durable volume on Hermes Cloud, so a - marker survives the machine restart that the drain-gated action (update, - migrate, env edit) ends in; honouring it would park the fresh gateway in - ``draining`` forever. A marker is stamped with this instantiation's epoch - and ignored only on a *definite* mismatch. - * expiry: a same-epoch orphan (action completed without a restart, writer - never cancelled) is ignored once ``requested_at`` is older than - :data:`DRAIN_REQUEST_MAX_AGE_SECONDS`. Re-calling :func:`write_drain_request` - refreshes the timestamp — the sanctioned keep-alive for a long drain. - -Reading never raises: a malformed/half-written file reads as "present but -contentless" (``{}``), which still counts as drain-active — fail-safe toward -quiescing. Both staleness checks only reject on a *definite* verdict; a marker -with no epoch/timestamp, or a host without ``/proc``, degrades to the -presence-only behaviour and never fails closed. +No control channel exists into a running gateway, so begin/cancel-drain writes +(or removes) ``{HERMES_HOME}/.drain_request.json`` and a gateway watcher reacts; +an ACTIVE marker means ``gateway_state -> "draining"``. Two lenient staleness +signals (either suffices): epoch mismatch (HERMES_HOME is a durable volume on +Hermes Cloud, so a marker survives the restart a drain-gated action ends in and +would park the fresh gateway in ``draining`` forever) and expiry (same-epoch +orphan past :data:`DRAIN_REQUEST_MAX_AGE_SECONDS`; re-writing refreshes it). +Reading never raises: a malformed file reads as ``{}`` — still drain-active +(fail-safe toward quiescing). Staleness rejects only on a *definite* verdict. """ from __future__ import annotations +import contextlib import functools import json import logging @@ -40,21 +20,18 @@ from datetime import datetime, timezone from pathlib import Path from typing import Any, Optional +from gateway.memory_status import _parse_iso from hermes_constants import get_hermes_home from utils import atomic_json_write -import contextlib _log = logging.getLogger(__name__) _DRAIN_REQUEST_FILENAME = ".drain_request.json" - -# Drain-gated lifecycle actions complete in minutes; an hour bounds the wedge a -# leaked marker can cause. Long drains refresh the marker instead of raising this. +# Drain-gated lifecycle actions complete in minutes; an hour bounds the wedge a leaked +# marker can cause. Long drains refresh the marker instead of raising this. DRAIN_REQUEST_MAX_AGE_SECONDS = 3600.0 - -# Dedup for the expired-marker warning: the watcher re-reads every second and an -# expired orphan sits on disk until removed. Keyed by ``requested_at`` so a -# keep-alive re-write that later expires again logs again. +# Dedup for the expired-marker warning (the watcher re-reads every second); keyed by +# ``requested_at`` so a keep-alive re-write that later expires logs again. _expiry_logged_for: Optional[str] = None @@ -62,62 +39,40 @@ _expiry_logged_for: Optional[str] = None def current_instantiation_epoch() -> str: """Identity of THIS container / VM instantiation (":"). - Stable for the life of PID 1 (so an s6 respawn of just the gateway, or a - host ``hermes gateway restart`` under systemd/launchd, keeps honouring an - in-flight drain) but changes whenever the machine is recreated: boot_id - changes on a VM/microVM reboot, PID 1's start time on a plain - ``docker restart`` (same host kernel, brand-new ``/init``). - - Returns ``""`` when neither source is readable (non-Linux, no ``/proc``), - which disables the epoch check downstream — never fail-closed. + Stable for the life of PID 1 (a gateway-only respawn keeps honouring an + in-flight drain) but changes when the machine is recreated: boot_id on a VM + reboot, PID 1's start time on ``docker restart``. ``""`` when neither is + readable (non-Linux, no ``/proc``) disables the epoch check — never fail-closed. """ - boot_id = "" + boot_id = pid1_start = "" with contextlib.suppress(OSError): boot_id = Path("/proc/sys/kernel/random/boot_id").read_text(encoding="utf-8").strip() - - pid1_start = "" - try: - # " () ...": comm may contain spaces/parens, so split - # on the LAST ')'. starttime is field 22 (1-indexed) = tail index 19. - stat = Path("/proc/1/stat").read_text(encoding="utf-8") - pid1_start = stat.rsplit(")", 1)[1].split()[19] - except (OSError, IndexError): - pass - - if not boot_id and not pid1_start: - return "" - return f"{boot_id}:{pid1_start}" + with contextlib.suppress(OSError, IndexError): + # " () ...": comm may contain spaces/parens, so split on the + # LAST ')'. starttime is field 22 (1-indexed) = tail index 19. + pid1_start = Path("/proc/1/stat").read_text(encoding="utf-8").rsplit(")", 1)[1].split()[19] + return f"{boot_id}:{pid1_start}" if (boot_id or pid1_start) else "" def drain_request_path(home: Optional[Path] = None) -> Path: """Absolute path to the drain-request marker, respecting HERMES_HOME.""" - base = home if home is not None else get_hermes_home() - return Path(base) / _DRAIN_REQUEST_FILENAME + return Path(home if home is not None else get_hermes_home()) / _DRAIN_REQUEST_FILENAME def write_drain_request( - *, - principal: str = "drain-control", - suppress_notification: bool = False, - home: Optional[Path] = None, + *, principal: str = "drain-control", suppress_notification: bool = False, home: Optional[Path] = None ) -> dict[str, Any]: - """Write the begin-drain marker atomically. Returns the payload written. + """Write the begin-drain marker atomically; returns the payload. - Idempotent: re-writing refreshes ``requested_at`` (keep-alive past the - max-age). ``suppress_notification`` asks the shutdown that ends this drain - to skip ONLY the home-channel "gateway shutting down" broadcast (the - per-active-session interrupt ping is never suppressed); the policy of - which drains are quiet lives entirely in the caller. Defaults False so - legacy/operator drains behave exactly as before. The marker is stamped - with :func:`current_instantiation_epoch` so a copy surviving a machine - restart on the durable volume can be recognised as stale. + Re-writing refreshes ``requested_at`` (keep-alive past the max-age). + ``suppress_notification`` skips ONLY the home-channel "gateway shutting down" + broadcast (the per-session interrupt ping is never suppressed); which drains + are quiet is the caller's policy. Stamped with the instantiation epoch so a + copy surviving a machine restart on the durable volume reads as stale. """ payload = { - "action": "drain", - "requested_at": datetime.now(timezone.utc).isoformat(), - "principal": principal, - "epoch": current_instantiation_epoch(), - "suppress_notification": bool(suppress_notification), + "action": "drain", "requested_at": datetime.now(timezone.utc).isoformat(), "principal": principal, + "epoch": current_instantiation_epoch(), "suppress_notification": bool(suppress_notification), } atomic_json_write(drain_request_path(home), payload) return payload @@ -129,65 +84,43 @@ def clear_drain_request(*, home: Optional[Path] = None) -> bool: try: path.unlink() return True - except FileNotFoundError: - return False except OSError as e: - _log.warning("drain-control: failed to remove %s: %s", path, e) + if not isinstance(e, FileNotFoundError): + _log.warning("drain-control: failed to remove %s: %s", path, e) return False -def _marker_epoch_is_stale(body: dict[str, Any]) -> bool: - """True iff ``body``'s epoch is a *definite* mismatch (both epochs known and differ).""" - current = current_instantiation_epoch() - marker_epoch = body.get("epoch") - return bool(current and marker_epoch and marker_epoch != current) - - def _marker_is_expired(body: dict[str, Any]) -> bool: """True iff ``requested_at`` parses AND is older than the max-age. - Missing/unparseable timestamps and future-dated ones (clock skew) are - honoured. The expiry is logged once per marker, not per poll: this path - only fires when a writer leaked a marker, and the warning is the - operator's breadcrumb. + Missing/unparseable and future-dated (clock skew) timestamps are honoured. + Logged once per marker, not per poll — the operator's breadcrumb for a leak. """ global _expiry_logged_for raw = body.get("requested_at") - if not isinstance(raw, str) or not raw: + requested_at = _parse_iso(raw) + if requested_at is None: return False - try: - requested_at = datetime.fromisoformat(raw) - except ValueError: - return False - if requested_at.tzinfo is None: - requested_at = requested_at.replace(tzinfo=timezone.utc) age = (datetime.now(timezone.utc) - requested_at).total_seconds() if age <= DRAIN_REQUEST_MAX_AGE_SECONDS: return False if _expiry_logged_for != raw: _expiry_logged_for = raw _log.warning( - "drain-control: ignoring expired drain marker (requested_at=%s, " - "age=%.0fs > max %.0fs, principal=%s) — the drain that wrote it " - "was never cancelled; treating as stale so the gateway keeps " - "accepting turns.", - raw, - age, - DRAIN_REQUEST_MAX_AGE_SECONDS, - body.get("principal"), + "drain-control: ignoring expired drain marker (requested_at=%s, age=%.0fs > max %.0fs, principal=%s) " + "— the drain that wrote it was never cancelled; treating as stale so the gateway keeps accepting turns.", + raw, age, DRAIN_REQUEST_MAX_AGE_SECONDS, body.get("principal"), ) return True -def _marker_is_stale(body: dict[str, Any]) -> bool: - """True iff the marker is definitely from a drain that is already over.""" - return _marker_epoch_is_stale(body) or _marker_is_expired(body) - - def _active_drain_body(home: Optional[Path]) -> Optional[dict[str, Any]]: - """Marker body if present AND not stale, else None.""" + """Marker body if present AND not stale (definite epoch mismatch or expired), else None.""" body = read_drain_request(home=home) - if body is None or _marker_is_stale(body): + if body is None: + return None + current, marker_epoch = current_instantiation_epoch(), body.get("epoch") + if (current and marker_epoch and marker_epoch != current) or _marker_is_expired(body): return None return body @@ -198,12 +131,10 @@ def drain_requested(*, home: Optional[Path] = None) -> bool: def drain_notification_suppressed(*, home: Optional[Path] = None) -> bool: - """True iff an ACTIVE drain marker explicitly asks to suppress the shutdown broadcast. + """True iff an ACTIVE marker asks to suppress the shutdown broadcast. - Uses the same activeness rule as :func:`drain_requested`, so an orphaned - marker can never silence a fresh gateway's legitimate broadcast. A legacy - marker without the field or a contentless ``{}`` body reads as False - (fail toward the louder behaviour). + Same activeness rule as :func:`drain_requested`, so an orphan can never silence + a fresh gateway; a marker without the field reads False (fail toward louder). """ body = _active_drain_body(home) return bool(body and body.get("suppress_notification")) @@ -214,10 +145,9 @@ def read_drain_request(*, home: Optional[Path] = None) -> Optional[dict[str, Any path = drain_request_path(home) try: raw = path.read_text(encoding="utf-8") - except FileNotFoundError: - return None except OSError as e: - _log.warning("drain-control: failed to read %s: %s", path, e) + if not isinstance(e, FileNotFoundError): + _log.warning("drain-control: failed to read %s: %s", path, e) return None try: data = json.loads(raw) diff --git a/gateway/hooks.py b/gateway/hooks.py index dcf29962a5..c0a84aa67a 100644 --- a/gateway/hooks.py +++ b/gateway/hooks.py @@ -1,30 +1,18 @@ -""" -Event Hook System +"""Event hook system: fires handlers at gateway lifecycle points. -Fires handlers at gateway lifecycle points. Hooks are discovered from -~/.hermes/hooks// directories, each containing HOOK.yaml (name, -description, events) and handler.py (``def handle(event_type, context)``, -sync or async). Handler errors are logged and never block the pipeline. - -Events: - gateway:startup, session:start, session:end (user ran /new or /reset), - session:reset, agent:start, agent:step (each tool-loop turn), agent:end, - command:* (any slash command; wildcard match). - -Context passed to ``agent:start`` / ``agent:end``: - platform, user_id, chat_id, thread_id (Telegram forum-topic / thread root - id as string; empty when not in a thread), chat_type ("dm" | "group" | - "forum" | ""), session_id, message (truncated to 500 chars). -``agent:end`` adds: response (truncated to 500 chars), model, provider. - -Handlers posting a follow-up into the same Telegram forum-topic should pass -``message_thread_id=int(thread_id)`` when ``chat_type == "forum"`` and -``thread_id`` is non-empty. +Hooks live in ~/.hermes/hooks// with HOOK.yaml (name, description, events) and +handler.py (``def handle(event_type, context)``, sync or async); errors never block +the pipeline. Events: gateway:startup, session:start/end/reset, agent:start, +agent:step (each tool-loop turn), agent:end, command:* (wildcard). agent:* context: +platform, user_id, chat_id, thread_id ("" outside a thread), chat_type +("dm"|"group"|"forum"|""), session_id, message (500 chars); agent:end adds response, +model, provider. Forum follow-ups pass ``message_thread_id=int(thread_id)``. """ import asyncio import importlib.util import sys +from pathlib import Path from typing import Any, Callable, Dict, List, Optional import yaml @@ -35,6 +23,42 @@ from hermes_cli.config import get_hermes_home HOOKS_DIR = get_hermes_home() / "hooks" +def _skip(name: str, reason: str) -> None: + print(f"[hooks] Skipping {name}: {reason}", flush=True) + + +def _load_hook_dir(hook_dir: Path) -> Optional[tuple]: + """``(name, events, handle_fn, description)`` for a valid hook dir, else None (reason printed).""" + manifest_path, handler_path = hook_dir / "HOOK.yaml", hook_dir / "handler.py" + if not manifest_path.exists() or not handler_path.exists(): + return None + manifest = yaml.safe_load(manifest_path.read_text(encoding="utf-8")) + if not manifest or not isinstance(manifest, dict): + return _skip(hook_dir.name, "invalid HOOK.yaml") + hook_name = manifest.get("name", hook_dir.name) + events = manifest.get("events", []) + if not events: + return _skip(hook_name, "no events declared") + # Register in sys.modules BEFORE exec_module so Pydantic/dataclass forward references + # (``from __future__ import annotations``) resolve; otherwise a handler declaring a + # BaseModel fails at first dispatch with "TypeAdapter ... is not fully defined". + module_name = f"hermes_hook_{hook_name}" + spec = importlib.util.spec_from_file_location(module_name, handler_path) + if spec is None or spec.loader is None: + return _skip(hook_name, "could not load handler.py") + module = importlib.util.module_from_spec(spec) + sys.modules[module_name] = module + try: + spec.loader.exec_module(module) + except Exception: + sys.modules.pop(module_name, None) + raise + handle_fn = getattr(module, "handle", None) + if handle_fn is None: + return _skip(hook_name, "no 'handle' function found") + return hook_name, events, handle_fn, manifest.get("description", "") + + class HookRegistry: """Discovers, loads, and fires event hooks.""" @@ -48,81 +72,33 @@ class HookRegistry: def _register_builtin_hooks(self) -> None: """Extension point for always-on built-in hooks; currently none shipped.""" - return def discover_and_load(self) -> None: """Register built-in hooks, then load every valid hook dir under HOOKS_DIR.""" self._register_builtin_hooks() - if not HOOKS_DIR.exists(): return - for hook_dir in sorted(HOOKS_DIR.iterdir()): if not hook_dir.is_dir(): continue - - manifest_path = hook_dir / "HOOK.yaml" - handler_path = hook_dir / "handler.py" - - if not manifest_path.exists() or not handler_path.exists(): - continue - try: - manifest = yaml.safe_load(manifest_path.read_text(encoding="utf-8")) - if not manifest or not isinstance(manifest, dict): - print(f"[hooks] Skipping {hook_dir.name}: invalid HOOK.yaml", flush=True) - continue - - hook_name = manifest.get("name", hook_dir.name) - events = manifest.get("events", []) - if not events: - print(f"[hooks] Skipping {hook_name}: no events declared", flush=True) - continue - - # Register in sys.modules BEFORE exec_module so Pydantic/dataclass - # forward references (from ``from __future__ import annotations``) - # resolve; otherwise a handler declaring a BaseModel fails at first - # dispatch with "TypeAdapter ... is not fully defined". - module_name = f"hermes_hook_{hook_name}" - spec = importlib.util.spec_from_file_location(module_name, handler_path) - if spec is None or spec.loader is None: - print(f"[hooks] Skipping {hook_name}: could not load handler.py", flush=True) - continue - - module = importlib.util.module_from_spec(spec) - sys.modules[module_name] = module - try: - spec.loader.exec_module(module) - except Exception: - sys.modules.pop(module_name, None) - raise - - handle_fn = getattr(module, "handle", None) - if handle_fn is None: - print(f"[hooks] Skipping {hook_name}: no 'handle' function found", flush=True) - continue - - for event in events: - self._handlers.setdefault(event, []).append(handle_fn) - - self._loaded_hooks.append({ - "name": hook_name, - "description": manifest.get("description", ""), - "events": events, - "path": str(hook_dir), - }) - - print(f"[hooks] Loaded hook '{hook_name}' for events: {events}", flush=True) - + loaded = _load_hook_dir(hook_dir) except Exception as e: print(f"[hooks] Error loading hook {hook_dir.name}: {e}", flush=True) + continue + if loaded is None: + continue + hook_name, events, handle_fn, description = loaded + for event in events: + self._handlers.setdefault(event, []).append(handle_fn) + self._loaded_hooks.append( + {"name": hook_name, "description": description, "events": events, "path": str(hook_dir)} + ) + print(f"[hooks] Loaded hook '{hook_name}' for events: {events}", flush=True) def _resolve_handlers(self, event_type: str) -> List[Callable]: - """Exact-match handlers first, then ``:*`` wildcard handlers. - - A handler registered for a bare base type ("agent") does NOT fire for - "agent:start" — only exact matches and explicit wildcards. - """ + """Exact-match handlers first, then ``:*`` wildcards. A bare base type + ("agent") does NOT fire for "agent:start" — only exact matches and explicit wildcards.""" handlers = list(self._handlers.get(event_type, [])) if ":" in event_type: handlers.extend(self._handlers.get(f"{event_type.split(':')[0]}:*", [])) @@ -132,26 +108,16 @@ class HookRegistry: """Fire all handlers for an event, discarding return values.""" await self.emit_collect(event_type, context) - async def emit_collect( - self, - event_type: str, - context: Optional[Dict[str, Any]] = None, - ) -> List[Any]: - """Fire handlers and return their non-None return values in order. - - Used for decision-style hooks (e.g. ``command:`` policies that - allow/deny/rewrite a command). A failing handler is logged and does not - abort the remaining handlers. - """ + async def emit_collect(self, event_type: str, context: Optional[Dict[str, Any]] = None) -> List[Any]: + """Fire handlers and return their non-None return values in order (decision-style + hooks, e.g. ``command:`` policies). A failing handler is logged, not fatal.""" if context is None: context = {} - results: List[Any] = [] for fn in self._resolve_handlers(event_type): try: result = fn(event_type, context) - if asyncio.iscoroutine(result): # sync and async handlers both supported - result = await result + result = await result if asyncio.iscoroutine(result) else result # sync or async handlers if result is not None: results.append(result) except Exception as e: diff --git a/gateway/hosted_room_discussion.py b/gateway/hosted_room_discussion.py index 3118e1c6ad..9273ca7ff6 100644 --- a/gateway/hosted_room_discussion.py +++ b/gateway/hosted_room_discussion.py @@ -1,20 +1,15 @@ """Deterministic policy for same-gateway hosted-room Discussions. -Translates a frozen local member roster plus the complete typed room log into -one next driver task. Pure: no I/O, no workers, no transport or model -knowledge. Callers persist the task with :mod:`gateway.hosted_room_driver` -and append publication plans with :mod:`gateway.hosted_rooms`. - -The driver payload is deliberately not widened: Discussion coordinates live -in deterministic ``TaskIdentity`` values and typed terminal events, so a -restart can reconstruct a task from durable state. Callers must reconcile -terminal driver rows into publication plans before asking for the next task. +Pure (no I/O, transport or model knowledge): a frozen local roster plus the complete +typed room log yields at most one next driver task. Discussion coordinates live in +deterministic ``TaskIdentity`` values and typed terminal events rather than a widened +driver payload, so a restart reconstructs tasks from durable state. Callers must +reconcile terminal driver rows into publication plans before asking for the next task. """ from __future__ import annotations import hashlib -import json import re from collections.abc import Iterable, Mapping, Sequence from dataclasses import dataclass @@ -24,6 +19,7 @@ from typing import Any, Literal from gateway import hosted_room_driver as driver from gateway import hosted_rooms from gateway import hosted_rooms_common as common +from gateway.hosted_rooms_common import compact_json MAX_DISCUSSION_MEMBERS = 6 @@ -42,29 +38,22 @@ _MENTION_RE = re.compile(r"@([A-Za-z0-9][A-Za-z0-9._:-]*)", re.IGNORECASE) _TURN_ID_RE = re.compile( r"^d(?P[1-9][0-9]*)\.r(?P[0-2])\." r"p(?P[0-5])\.s(?P[1-9][0-9]*)\." - r"m(?P[0-9a-f]{24})$" -) + r"m(?P[0-9a-f]{24})$") _LOCAL_TARGET_FIELDS = frozenset({"kind", "profile"}) _PEER_TARGET_FIELDS = frozenset({"kind", "peer_id", "installation_id", "profile", "capability_digest"}) _REMOTE_MEMBER_FIELDS = frozenset({ - "connectionId", "connectionKind", "connectionLabel", "connection_id", - "connection_kind", "connection_label", "remoteSource", "route", - "sourceMissing", "sourceReachable", "sourceScoped", "targetProfile", - "target_profile", + "connectionId", "connectionKind", "connectionLabel", "connection_id", "connection_kind", "connection_label", + "remoteSource", "route", "sourceMissing", "sourceReachable", "sourceScoped", "targetProfile", "target_profile", }) _USER_PAYLOAD_FIELDS = frozenset({"text", "thread_id"}) -_TURN_COORDINATE_FIELDS = frozenset({ - "discussion_event_id", "member_id", "member_index", "round_index", - "task_id", "thread_id", "turn_id", -}) +_TURN_COORDINATE_FIELDS = frozenset( + {"discussion_event_id", "member_id", "member_index", "round_index", "task_id", "thread_id", "turn_id", }) _MEMBER_MESSAGE_FIELDS = _TURN_COORDINATE_FIELDS | {"text"} _TERMINAL_COMMON_FIELDS = _TURN_COORDINATE_FIELDS | {"seen_through_seq"} _TERMINAL_EXTRA_FIELDS = { - "turn.settled": frozenset({"message_event_id", "passed"}), - "turn.failed": frozenset({"error"}), - "turn.cancelled": frozenset({"reason"}), - "turn.deferred": frozenset({"execution_generation", "reason"}), + "turn.settled": frozenset({"message_event_id", "passed"}), "turn.failed": frozenset({"error"}), + "turn.cancelled": frozenset({"reason"}), "turn.deferred": frozenset({"execution_generation", "reason"}), } _TERMINAL_OPTIONAL_FIELDS = {"turn.failed": frozenset({"reason_code"})} _TERMINAL_EVENT_KINDS = frozenset(_TERMINAL_EXTRA_FIELDS) @@ -72,15 +61,12 @@ _TERMINAL_EVENT_KINDS = frozenset(_TERMINAL_EXTRA_FIELDS) _GATEWAY_EVENT_FIELDS = { "room.activity": ( frozenset({"status", "reason_code", "thread_id", "discussion_event_id"}), - ("reason_code", "thread_id", "discussion_event_id"), - ), - "room.stop_requested": (frozenset({"cancel_id"}), ("cancel_id",)), -} + ("reason_code", "thread_id", "discussion_event_id")), + "room.stop_requested": (frozenset({"cancel_id"}), ("cancel_id",))} _EPOCH_STAMPED_KINDS = _TERMINAL_EVENT_KINDS | {"message.member", *_GATEWAY_EVENT_FIELDS} -class DiscussionPolicyError(ValueError): - """Base class for invalid policy input or unreconstructable state.""" +class DiscussionPolicyError(ValueError): """Base class for invalid policy input or unreconstructable state.""" class DiscussionValidationError(DiscussionPolicyError): @@ -155,8 +141,7 @@ class EventPlan: "room_id": room_id, "event_id": self.event_id, "kind": self.kind, "actor": dict(self.actor), "payload": dict(self.payload), "authority_gateway_id": self.authority_gateway_id, - "authority_epoch": self.authority_epoch, - } + "authority_epoch": self.authority_epoch} @dataclass(frozen=True) @@ -178,22 +163,20 @@ class _ValidatedEvent: payload: Mapping[str, Any] -_identifier = partial( - common.identifier, error=DiscussionValidationError, max_chars=driver.MAX_IDENTIFIER_CHARS -) +_identifier = partial(common.identifier, error=DiscussionValidationError, max_chars=driver.MAX_IDENTIFIER_CHARS) _exact_fields = partial(common.exact_fields, error=DiscussionValidationError) +_bounded_int = partial(common.bounded_int, error=DiscussionValidationError) def _positive_int(value: Any, *, label: str) -> int: - return common.positive_int( - value, error=DiscussionValidationError, message=f"{label} must be a positive integer" - ) + return _bounded_int(value, message=f"{label} must be a positive integer", low=1) def _zero_based_int(value: Any, *, label: str, maximum: int) -> int: - if isinstance(value, bool) or not isinstance(value, int) or not 0 <= value <= maximum: - raise DiscussionValidationError(f"{label} must be an integer between 0 and {maximum}") - return value + message = f"{label} must be an integer between 0 and {maximum}" + return _bounded_int(value, message=message, high=maximum) + +_text = partial(common.text, error=DiscussionValidationError) def _peer_target(member: DiscussionMember) -> Mapping[str, Any] | None: @@ -204,25 +187,15 @@ def _peer_target(member: DiscussionMember) -> Mapping[str, Any] | None: def validate_user_payload(value: Any) -> dict[str, Any]: """Validate and normalize the exact ``message.user`` Discussion payload.""" payload = _exact_fields(value, label="user payload", required=_USER_PAYLOAD_FIELDS) - text = payload["text"] - if not isinstance(text, str): - raise DiscussionValidationError("user payload text must be a string") - text = text.strip() - if not text: - raise DiscussionValidationError("user payload text must not be empty") - if len(text.encode("utf-8")) > MAX_USER_TEXT_BYTES: - raise DiscussionValidationError("user payload text is too large") + text = _text(payload["text"], label="user payload text", max_bytes=MAX_USER_TEXT_BYTES) return {"text": text, "thread_id": _identifier(payload["thread_id"], label="thread_id")} -def _validate_member_target( - value: Any, *, profile: str, known_profiles: set[str], index: int -) -> dict[str, Any]: +def _validate_member_target(value: Any, *, profile: str, known_profiles: set[str], index: int) -> dict[str, Any]: if value is None: if profile not in known_profiles: raise DiscussionValidationError( - f"member {index} profile '{profile}' is not local to this gateway" - ) + f"member {index} profile '{profile}' is not local to this gateway") return {"kind": "local", "profile": profile} if not isinstance(value, Mapping): raise DiscussionValidationError(f"member {index} target must be an object") @@ -231,19 +204,16 @@ def _validate_member_target( raise DiscussionValidationError(f"member {index} target kind must be local or peer") target = _exact_fields( value, label=f"member {index} {kind} target", - required=_LOCAL_TARGET_FIELDS if kind == "local" else _PEER_TARGET_FIELDS, - ) + required=_LOCAL_TARGET_FIELDS if kind == "local" else _PEER_TARGET_FIELDS) target_profile = _identifier(target["profile"], label=f"member {index} target profile") if kind == "local": if target_profile != profile or profile not in known_profiles: raise DiscussionValidationError( - f"member {index} local target does not match a local profile" - ) + f"member {index} local target does not match a local profile") return {"kind": "local", "profile": profile} if target_profile != profile: raise DiscussionValidationError( - f"member {index} peer target profile does not match member profile" - ) + f"member {index} peer target profile does not match member profile") capability_digest = target["capability_digest"] if not isinstance(capability_digest, str) or not re.fullmatch(r"[0-9a-f]{64}", capability_digest): raise DiscussionValidationError(f"member {index} capability_digest must be a sha256 digest") @@ -252,8 +222,7 @@ def _validate_member_target( "peer_id": _identifier(target["peer_id"], label=f"member {index} peer_id"), "installation_id": _identifier(target["installation_id"], label=f"member {index} installation_id"), "profile": target_profile, - "capability_digest": capability_digest, - } + "capability_digest": capability_digest} def validate_roster(value: Any, *, local_profiles: Iterable[str]) -> tuple[DiscussionMember, ...]: @@ -263,8 +232,7 @@ def validate_roster(value: Any, *, local_profiles: Iterable[str]) -> tuple[Discu if not MIN_DISCUSSION_MEMBERS <= len(value) <= MAX_DISCUSSION_MEMBERS: raise DiscussionValidationError( f"members must contain between {MIN_DISCUSSION_MEMBERS} and " - f"{MAX_DISCUSSION_MEMBERS} entries" - ) + f"{MAX_DISCUSSION_MEMBERS} entries") known_profiles = {_identifier(profile, label="local profile") for profile in local_profiles} members: list[DiscussionMember] = [] @@ -278,19 +246,16 @@ def validate_roster(value: Any, *, local_profiles: Iterable[str]) -> tuple[Discu remote_fields = frozenset(raw) & _REMOTE_MEMBER_FIELDS if remote_fields: raise DiscussionValidationError( - f"member {index} contains cross-gateway fields: {', '.join(sorted(remote_fields))}" - ) + f"member {index} contains cross-gateway fields: {', '.join(sorted(remote_fields))}") member = _exact_fields( raw, label=f"member {index}", required=frozenset({"member_id", "profile", "handle"}), - optional=frozenset({"display_name", "target"}), - ) + optional=frozenset({"display_name", "target"})) member_id = _identifier(member["member_id"], label=f"member {index} id") profile = _identifier(member["profile"], label=f"member {index} profile") handle = _identifier(member["handle"], label=f"member {index} handle") target = _validate_member_target( - member.get("target"), profile=profile, known_profiles=known_profiles, index=index - ) + member.get("target"), profile=profile, known_profiles=known_profiles, index=index) display_name = member.get("display_name", "") if not isinstance(display_name, str): raise DiscussionValidationError(f"member {index} display_name must be a string") @@ -298,24 +263,20 @@ def validate_roster(value: Any, *, local_profiles: Iterable[str]) -> tuple[Discu if len(display_name) > hosted_rooms.MAX_ACTOR_LABEL_CHARS: raise DiscussionValidationError(f"member {index} display_name is too long") - target_key = json.dumps(target, sort_keys=True, separators=(",", ":")).casefold() + target_key = compact_json(target, ensure_ascii=False).casefold() target_message = ( "member profiles must be unique" if target.get("kind") == "local" - else "member targets must be unique" - ) + else "member targets must be unique") for key, seen, message in ( (target_key, targets, target_message), - (handle.casefold(), handles, - "member handles must be unique and cannot reserve @all or @everyone"), - (member_id.casefold(), member_ids, "member ids must be unique"), - ): + (handle.casefold(), handles, "member handles must be unique and cannot reserve @all or @everyone"), + (member_id.casefold(), member_ids, "member ids must be unique")): if key in seen: raise DiscussionValidationError(message) seen.add(key) members.append(DiscussionMember( member_id=member_id, profile=profile, handle=handle, - display_name=display_name, target=target, - )) + display_name=display_name, target=target)) return tuple(members) @@ -336,9 +297,7 @@ def validate_room(value: Any, *, local_profiles: Iterable[str]) -> DiscussionRoo authority_epoch = _positive_int(value.get("authority_epoch"), label="authority_epoch") members = validate_roster(value.get("members"), local_profiles=local_profiles) return DiscussionRoom( - room_id=room_id, name=name, members=members, - gateway_id=gateway_id, authority_epoch=authority_epoch, - ) + room_id=room_id, name=name, members=members, gateway_id=gateway_id, authority_epoch=authority_epoch) def is_pass_text(value: Any) -> bool: @@ -367,8 +326,7 @@ def resolve_mentions( def _unaddressed_member_mentions( - messages: Sequence[_ValidatedEvent], room: DiscussionRoom -) -> tuple[DiscussionMember, ...]: + messages: Sequence[_ValidatedEvent], room: DiscussionRoom) -> tuple[DiscussionMember, ...]: """Return peers explicitly cited by a Bot and not heard from afterward.""" cited_at: dict[str, int] = {} last_post_at: dict[str, int] = {} @@ -384,8 +342,7 @@ def _unaddressed_member_mentions( return tuple( member for member in room.members if member.member_id in cited_at - and last_post_at.get(member.member_id, 0) <= cited_at[member.member_id] - ) + and last_post_at.get(member.member_id, 0) <= cited_at[member.member_id]) def _require_gateway_actor(actor: Mapping[str, Any], room: DiscussionRoom, message: str) -> None: @@ -395,10 +352,10 @@ def _require_gateway_actor(actor: Mapping[str, Any], room: DiscussionRoom, messa def _member_by_id(room: DiscussionRoom, member_id: Any) -> DiscussionMember: normalized = _identifier(member_id, label="member_id") - for member in room.members: - if member.member_id == normalized: - return member - raise DiscussionValidationError(f"unknown Discussion member '{normalized}'") + member = next((m for m in room.members if m.member_id == normalized), None) + if member is None: + raise DiscussionValidationError(f"unknown Discussion member '{normalized}'") + return member def _validate_turn_coordinates(payload: Mapping[str, Any], room: DiscussionRoom) -> None: @@ -414,8 +371,7 @@ def _validate_turn_coordinates(payload: Mapping[str, Any], room: DiscussionRoom) def _validate_user_event( - kind: str, payload: Mapping[str, Any], actor: Mapping[str, Any], room: DiscussionRoom -) -> Mapping[str, Any]: + kind: str, payload: Mapping[str, Any], actor: Mapping[str, Any], room: DiscussionRoom) -> Mapping[str, Any]: payload = validate_user_payload(payload) if actor.get("kind") != "user": raise DiscussionValidationError("message.user requires a user actor") @@ -423,8 +379,7 @@ def _validate_user_event( def _validate_member_message( - kind: str, payload: Mapping[str, Any], actor: Mapping[str, Any], room: DiscussionRoom -) -> Mapping[str, Any]: + kind: str, payload: Mapping[str, Any], actor: Mapping[str, Any], room: DiscussionRoom) -> Mapping[str, Any]: _exact_fields(payload, label="message.member payload", required=_MEMBER_MESSAGE_FIELDS) _validate_turn_coordinates(payload, room) text = payload.get("text") @@ -433,9 +388,7 @@ def _validate_member_message( member = _member_by_id(room, payload.get("member_id")) peer = _peer_target(member) expected = { - "kind": "member", - "id": member.member_id, - "profile": member.profile, + "kind": "member", "id": member.member_id, "profile": member.profile, "connection_id": peer.get("peer_id") if peer else None, } if any(actor.get(key) != value for key, value in expected.items()): @@ -444,13 +397,11 @@ def _validate_member_message( def _validate_terminal_event( - kind: str, payload: Mapping[str, Any], actor: Mapping[str, Any], room: DiscussionRoom -) -> Mapping[str, Any]: + kind: str, payload: Mapping[str, Any], actor: Mapping[str, Any], room: DiscussionRoom) -> Mapping[str, Any]: _exact_fields( payload, label=f"{kind} payload", required=_TERMINAL_COMMON_FIELDS | _TERMINAL_EXTRA_FIELDS[kind], - optional=_TERMINAL_OPTIONAL_FIELDS.get(kind, frozenset()), - ) + optional=_TERMINAL_OPTIONAL_FIELDS.get(kind, frozenset())) _validate_turn_coordinates(payload, room) _positive_int(payload.get("seen_through_seq"), label="seen_through_seq") if actor.get("connection_id") is not None: @@ -474,15 +425,12 @@ def _validate_terminal_event( from tools.bot_failure_reasons import ALL_REASONS if payload["reason_code"] not in ALL_REASONS: - raise DiscussionValidationError( - "turn.failed reason_code must use the shared failure vocabulary" - ) + raise DiscussionValidationError("turn.failed reason_code must use the shared failure vocabulary") return payload def _validate_gateway_event( - kind: str, payload: Mapping[str, Any], actor: Mapping[str, Any], room: DiscussionRoom -) -> Mapping[str, Any]: + kind: str, payload: Mapping[str, Any], actor: Mapping[str, Any], room: DiscussionRoom) -> Mapping[str, Any]: fields, identifier_fields = _GATEWAY_EVENT_FIELDS[kind] _exact_fields(payload, label=f"{kind} payload", required=fields) if kind == "room.activity" and payload.get("status") not in {"settled", "bounded"}: @@ -497,8 +445,7 @@ _EVENT_VALIDATORS = { "message.user": _validate_user_event, "message.member": _validate_member_message, **dict.fromkeys(_TERMINAL_EVENT_KINDS, _validate_terminal_event), - **dict.fromkeys(_GATEWAY_EVENT_FIELDS, _validate_gateway_event), -} + **dict.fromkeys(_GATEWAY_EVENT_FIELDS, _validate_gateway_event)} def _validate_event(raw: Any, *, room: DiscussionRoom, previous_seq: int) -> _ValidatedEvent: @@ -514,8 +461,7 @@ def _validate_event(raw: Any, *, room: DiscussionRoom, previous_seq: int) -> _Va for value, expected, message in ( (kind, str, "event kind must be a string"), (actor, Mapping, "event actor must be an object"), - (payload, Mapping, "event payload must be an object"), - ): + (payload, Mapping, "event payload must be an object")): if not isinstance(value, expected): raise DiscussionValidationError(message) if kind in _EPOCH_STAMPED_KINDS and raw.get("authority_epoch") != room.authority_epoch: @@ -526,9 +472,7 @@ def _validate_event(raw: Any, *, room: DiscussionRoom, previous_seq: int) -> _Va return _ValidatedEvent(raw=raw, seq=seq, event_id=event_id, kind=kind, actor=actor, payload=payload) -def _validated_events( - events: Sequence[Mapping[str, Any]], *, room: DiscussionRoom -) -> tuple[_ValidatedEvent, ...]: +def _validated_events(events: Sequence[Mapping[str, Any]], *, room: DiscussionRoom) -> tuple[_ValidatedEvent, ...]: validated: list[_ValidatedEvent] = [] previous_seq = 0 event_ids: set[str] = set() @@ -562,14 +506,12 @@ def _derive_member_watermarks(events: Sequence[_ValidatedEvent]) -> dict[tuple[s if previous is not None: if previous.kind != "turn.deferred": raise DiscussionValidationError( - f"task '{task_id}' has more than one terminal room event" - ) + f"task '{task_id}' has more than one terminal room event") if event.kind == "turn.deferred" and int( event.payload["execution_generation"] ) <= int(previous.payload["execution_generation"]): raise DiscussionValidationError( - f"task '{task_id}' deferral generation did not advance" - ) + f"task '{task_id}' deferral generation did not advance") terminal_by_task[task_id] = event key = (str(event.payload["thread_id"]), str(event.payload["member_id"])) watermark = int(event.payload["seen_through_seq"]) @@ -577,8 +519,7 @@ def _derive_member_watermarks(events: Sequence[_ValidatedEvent]) -> dict[tuple[s message = messages_by_id.get(str(event.payload["message_event_id"])) if message is None or any( message.payload.get(field) != event.payload.get(field) - for field in ("task_id", "member_id", "thread_id") - ): + for field in ("task_id", "member_id", "thread_id")): raise DiscussionValidationError("turn.settled references no matching member message") watermark = max(watermark, message.seq) watermarks[key] = max(watermarks.get(key, 0), watermark) @@ -586,10 +527,7 @@ def _derive_member_watermarks(events: Sequence[_ValidatedEvent]) -> dict[tuple[s def _member_digest(member: DiscussionMember) -> str: - target = json.dumps( - member.target or {"kind": "local", "profile": member.profile}, - sort_keys=True, separators=(",", ":"), - ) + target = compact_json(member.target or {"kind": "local", "profile": member.profile}, ensure_ascii=False) seed = f"{member.member_id}\0{member.profile}\0{member.handle}\0{target}" return hashlib.sha256(seed.encode("utf-8")).hexdigest()[:24] @@ -626,28 +564,22 @@ def _truncate_utf8_text(value: Any, *, max_bytes: int, suffix: str = "") -> str: def _build_prompt( *, room: DiscussionRoom, member: DiscussionMember, messages: Sequence[_ValidatedEvent], - watermark: int, seen_through_seq: int, -) -> str: - delta = [event for event in messages if watermark < event.seq <= seen_through_seq][ - -MAX_DISCUSSION_DELTA_LINES: - ] + watermark: int, seen_through_seq: int) -> str: + delta = [event for event in messages if watermark < event.seq <= seen_through_seq][- MAX_DISCUSSION_DELTA_LINES:] peers = ", ".join( - f"@{candidate.handle}" for candidate in room.members if candidate.member_id != member.member_id - ) + f"@{candidate.handle}" for candidate in room.members if candidate.member_id != member.member_id) opening = [ f'[Discussion: "{room.name}"] You are @{member.handle}, one participant ' f"with {peers or 'no other members'} and the user.", "", - "New messages in this thread since your last turn (oldest first):", - ] + "New messages in this thread since your last turn (oldest first):"] rules = [ "", "Rules for this Discussion:", "- Reply with one conversational message only when you have something new worth adding.", '- If you have nothing new to add, reply with exactly "(pass)".', "- Mention a teammate by handle to pull them into the next round; do not repeat points already made.", - "- Never reveal content from private conversations. Your reply is published verbatim.", - ] + "- Never reveal content from private conversations. Your reply is published verbatim."] fixed_bytes = len("\n".join([*opening, *rules]).encode("utf-8")) available = max(0, driver.MAX_PROMPT_BYTES - fixed_bytes - 1) selected: list[str] = [] @@ -674,54 +606,36 @@ def _build_prompt( def _make_task_plan( *, room: DiscussionRoom, discussion_event: _ValidatedEvent, member: DiscussionMember, - member_index: int, round_index: int, seen_through_seq: int, prompt: str, -) -> DiscussionTaskPlan: + member_index: int, round_index: int, seen_through_seq: int, prompt: str) -> DiscussionTaskPlan: turn_id = ( f"d{discussion_event.seq}.r{round_index}.p{member_index}." - f"s{seen_through_seq}.m{_member_digest(member)}" - ) - seed = json.dumps( - { - "discussion_event_id": discussion_event.event_id, - "member_id": member.member_id, - "member_index": member_index, - "prompt_sha256": hashlib.sha256(prompt.encode("utf-8")).hexdigest(), - "room_id": room.room_id, - "round_index": round_index, - "seen_through_seq": seen_through_seq, - "source_event_seq": discussion_event.seq, - "thread_id": discussion_event.payload["thread_id"], - }, - ensure_ascii=True, sort_keys=True, separators=(",", ":"), - ) + f"s{seen_through_seq}.m{_member_digest(member)}") + seed = compact_json({ + "discussion_event_id": discussion_event.event_id, "member_id": member.member_id, "member_index": member_index, + "prompt_sha256": hashlib.sha256(prompt.encode("utf-8")).hexdigest(), "room_id": room.room_id, + "round_index": round_index, "seen_through_seq": seen_through_seq, "source_event_seq": discussion_event.seq, + "thread_id": discussion_event.payload["thread_id"], + }) task_id = f"dtask:{hashlib.sha256(seed.encode('utf-8')).hexdigest()[:48]}" identity = driver.TaskIdentity( room_id=room.room_id, task_id=task_id, - thread_id=str(discussion_event.payload["thread_id"]), turn_id=turn_id, - ) + thread_id=str(discussion_event.payload["thread_id"]), turn_id=turn_id) payload = { - "target_member_id": member.member_id, - "target_profile": member.profile, - "prompt": prompt, + "target_member_id": member.member_id, "target_profile": member.profile, "prompt": prompt, "source_event_seq": discussion_event.seq, } return DiscussionTaskPlan( - identity=identity, payload=payload, discussion_event_id=discussion_event.event_id, - member=member, member_index=member_index, round_index=round_index, - seen_through_seq=seen_through_seq, - ) + identity=identity, payload=payload, discussion_event_id=discussion_event.event_id, member=member, + member_index=member_index, round_index=round_index, seen_through_seq=seen_through_seq) def _pending_discussion(validated: Sequence[_ValidatedEvent]) -> _ValidatedEvent | None: """Oldest latest-per-thread user message not stopped and not yet completed.""" - stopped_through_seq = max( - (event.seq for event in validated if event.kind == "room.stop_requested"), default=0 - ) + stopped_through_seq = max((event.seq for event in validated if event.kind == "room.stop_requested"), default=0) completed_discussion_ids = { str(event.payload["discussion_event_id"]) for event in validated - if event.kind == "room.activity" and event.payload.get("status") in {"settled", "bounded"} - } + if event.kind == "room.activity" and event.payload.get("status") in {"settled", "bounded"}} latest_by_thread: dict[str, _ValidatedEvent] = {} for event in validated: if event.kind == "message.user": @@ -729,8 +643,7 @@ def _pending_discussion(validated: Sequence[_ValidatedEvent]) -> _ValidatedEvent pending = [ event for event in sorted(latest_by_thread.values(), key=lambda item: item.seq) - if event.seq > stopped_through_seq and event.event_id not in completed_discussion_ids - ] + if event.seq > stopped_through_seq and event.event_id not in completed_discussion_ids] return pending[0] if pending else None @@ -742,8 +655,7 @@ def _thread_messages( committed_member_message_ids = { str(event.payload["message_event_id"]) for event in validated - if event.kind == "turn.settled" and event.payload.get("message_event_id") is not None - } + if event.kind == "turn.settled" and event.payload.get("message_event_id") is not None} # Publication writes the visible member message before the terminal event. # A crash in that gap leaves the message in the log, but it is not committed # policy input yet: ignoring it reproduces the original task coordinates so @@ -754,15 +666,12 @@ def _thread_messages( if event.payload.get("thread_id") == thread_id and ( event.kind == "message.user" - or (event.kind == "message.member" and event.event_id in committed_member_message_ids) - ) - ) + or (event.kind == "message.member" and event.event_id in committed_member_message_ids))) discussion_messages = tuple(event for event in thread_messages if event.seq >= discussion.seq) member_messages = tuple( event for event in thread_messages if event.kind == "message.member" - and event.payload.get("discussion_event_id") == discussion.event_id - ) + and event.payload.get("discussion_event_id") == discussion.event_id) return thread_messages, discussion_messages, member_messages @@ -772,20 +681,15 @@ def _effective_watermarks( watermarks = { (str(thread_id), str(member_id)): int(value) for (thread_id, member_id), value in (initial_watermarks or {}).items() - if int(value) >= 0 - } + if int(value) >= 0} for key, value in _derive_member_watermarks(validated).items(): watermarks[key] = max(watermarks.get(key, 0), value) return watermarks def plan_next_task( - room_value: Any, - events: Sequence[Mapping[str, Any]], - *, - local_profiles: Iterable[str], - initial_watermarks: Mapping[tuple[str, str], int] | None = None, -) -> DiscussionDecision: + room_value: Any, events: Sequence[Mapping[str, Any]], *, local_profiles: Iterable[str], + initial_watermarks: Mapping[tuple[str, str], int] | None = None) -> DiscussionDecision: """Replay the complete room log and return at most one next member task.""" room = validate_room(room_value, local_profiles=local_profiles) validated = _validated_events(events, room=room) @@ -795,8 +699,7 @@ def plan_next_task( thread_id = str(discussion.payload["thread_id"]) decide = partial( DiscussionDecision, discussion_event_id=discussion.event_id, - source_event_seq=discussion.seq, thread_id=thread_id, - ) + source_event_seq=discussion.seq, thread_id=thread_id) thread_messages, discussion_messages, member_messages = _thread_messages(validated, discussion) if len(member_messages) >= MAX_DISCUSSION_MESSAGES: @@ -806,8 +709,7 @@ def plan_next_task( (int(event.payload["round_index"]), str(event.payload["member_id"])): event for event in validated if event.kind in _TERMINAL_EVENT_KINDS - and event.payload.get("discussion_event_id") == discussion.event_id - } + and event.payload.get("discussion_event_id") == discussion.event_id} watermarks = _effective_watermarks(validated, initial_watermarks) seen_through_seq = max(event.seq for event in thread_messages) @@ -820,8 +722,7 @@ def plan_next_task( responders = ( resolve_mentions((str(discussion.payload["text"]),), room.members) if round_index == 0 - else _unaddressed_member_mentions(discussion_messages, room) - ) + else _unaddressed_member_mentions(discussion_messages, room)) for member_index, member in enumerate(_rotate(responders, round_index)): if (round_index, member.member_id) in terminals: continue @@ -830,12 +731,10 @@ def plan_next_task( continue prompt = _build_prompt( room=room, member=member, messages=thread_messages, - watermark=watermark, seen_through_seq=seen_through_seq, - ) + watermark=watermark, seen_through_seq=seen_through_seq) task = _make_task_plan( room=room, discussion_event=discussion, member=member, member_index=member_index, - round_index=round_index, seen_through_seq=seen_through_seq, prompt=prompt, - ) + round_index=round_index, seen_through_seq=seen_through_seq, prompt=prompt) return decide("task", "member_turn", task=task) if not any(int(event.payload["round_index"]) == round_index for event in member_messages): @@ -848,8 +747,7 @@ def plan_next_task( def reconstruct_task_plan( room_value: Any, events: Sequence[Mapping[str, Any]], task: Mapping[str, Any], - *, local_profiles: Iterable[str], -) -> DiscussionTaskPlan: + *, local_profiles: Iterable[str]) -> DiscussionTaskPlan: """Reconstruct and verify one persisted driver task after a restart.""" room = validate_room(room_value, local_profiles=local_profiles) validated = _validated_events(events, room=room) @@ -858,9 +756,7 @@ def reconstruct_task_plan( if not isinstance(identity, driver.TaskIdentity) or not isinstance(payload, Mapping): raise DiscussionReconstructionError("driver task has no valid identity or payload") required_payload = frozenset({"target_profile", "prompt", "source_event_seq"}) - if not required_payload <= frozenset(payload) or ( - frozenset(payload) - required_payload - {"target_member_id"} - ): + if not required_payload <= frozenset(payload) or (frozenset(payload) - required_payload - {"target_member_id"}): raise DiscussionReconstructionError("driver task payload shape changed") match = _TURN_ID_RE.fullmatch(identity.turn_id) if match is None: @@ -868,9 +764,7 @@ def reconstruct_task_plan( source_event_seq = int(match.group("source")) if payload.get("source_event_seq") != source_event_seq: raise DiscussionReconstructionError("task source event does not match turn_id") - discussion = next( - (e for e in validated if e.seq == source_event_seq and e.kind == "message.user"), None - ) + discussion = next((e for e in validated if e.seq == source_event_seq and e.kind == "message.user"), None) if discussion is None: raise DiscussionReconstructionError("task source user event is missing") if identity.room_id != room.room_id or identity.thread_id != discussion.payload["thread_id"]: @@ -880,16 +774,9 @@ def reconstruct_task_plan( member = next( ( candidate for candidate in room.members - if ( - candidate.member_id == target_member_id - if target_member_id is not None - else candidate.profile == profile - ) - ), - None, - ) - if member is not None and member.profile != profile: - member = None + if candidate.profile == profile + and (target_member_id is None or candidate.member_id == target_member_id)), + None) if member is None or _member_digest(member) != match.group("member"): raise DiscussionReconstructionError("task target member does not match turn_id") prompt = payload.get("prompt") @@ -898,15 +785,22 @@ def reconstruct_task_plan( if len(prompt.encode("utf-8")) > driver.MAX_PROMPT_BYTES: raise DiscussionReconstructionError("task prompt exceeds the driver limit") reconstructed = _make_task_plan( - room=room, discussion_event=discussion, member=member, - member_index=int(match.group("position")), round_index=int(match.group("round")), - seen_through_seq=int(match.group("seen")), prompt=prompt, - ) + room=room, discussion_event=discussion, member=member, member_index=int(match.group("position")), + round_index=int(match.group("round")), seen_through_seq=int(match.group("seen")), prompt=prompt) if reconstructed.identity != identity or dict(reconstructed.payload) != dict(payload): raise DiscussionReconstructionError("driver task failed deterministic reconstruction") return reconstructed +def _turn_coordinates(task: DiscussionTaskPlan) -> dict[str, Any]: + """The coordinate fields every member message and terminal event carries.""" + return { + "discussion_event_id": task.discussion_event_id, "member_id": task.member.member_id, + "member_index": task.member_index, "round_index": task.round_index, "task_id": task.identity.task_id, + "thread_id": task.identity.thread_id, "turn_id": task.identity.turn_id, + } + + def _terminal_text(result: Any, *, field: str, fallback: str) -> str: if isinstance(result, Mapping): value = result.get(field) @@ -927,8 +821,7 @@ def _settled_effects( ) -> tuple[dict[str, Any], list[EventPlan]]: text = _truncate_utf8_text( _terminal_text(result, field="text", fallback=""), - max_bytes=MAX_MEMBER_TEXT_BYTES, suffix=_TRUNCATED_REPLY_NOTICE, - ) + max_bytes=MAX_MEMBER_TEXT_BYTES, suffix=_TRUNCATED_REPLY_NOTICE) passed = is_pass_text(text) effects: list[EventPlan] = [] if not passed: @@ -940,18 +833,8 @@ def _settled_effects( member_actor["display_name"] = task.member.display_name effects.append(EventPlan( event_id=message_event_id, kind="message.member", actor=member_actor, - payload={ - "discussion_event_id": task.discussion_event_id, - "member_id": task.member.member_id, - "member_index": task.member_index, - "round_index": task.round_index, - "task_id": task.identity.task_id, - "text": text, - "thread_id": task.identity.thread_id, - "turn_id": task.identity.turn_id, - }, - authority_gateway_id=room.gateway_id, authority_epoch=room.authority_epoch, - )) + payload={**_turn_coordinates(task), "text": text}, + authority_gateway_id=room.gateway_id, authority_epoch=room.authority_epoch)) return {"message_event_id": None if passed else message_event_id, "passed": passed}, effects @@ -961,25 +844,20 @@ def _failed_effects(result: Any, **_: Any) -> tuple[dict[str, Any], list[EventPl supplied_reason = ( str(result.get("reason_code") or result.get("reason") or "").strip() - if isinstance(result, Mapping) else "" - ) + if isinstance(result, Mapping) else "") reason_code = supplied_reason if supplied_reason in ALL_REASONS else classify_agent_error(error_text) return {"error": error_text, "reason_code": reason_code}, [] -def _cancelled_effects( - result: Any, *, newer_same_thread: bool, **_: Any -) -> tuple[dict[str, Any], list[EventPlan]]: +def _cancelled_effects(result: Any, *, newer_same_thread: bool, ** _: Any) -> tuple[dict[str, Any], list[EventPlan]]: reason = ( "superseded_by_newer_user_event" if newer_same_thread - else _terminal_text(result, field="reason", fallback="member turn cancelled") - ) + else _terminal_text(result, field="reason", fallback="member turn cancelled")) return {"reason": reason}, [] def _deferred_effects( - result: Any, *, execution_generation: int | None, **_: Any -) -> tuple[dict[str, Any], list[EventPlan]]: + result: Any, *, execution_generation: int | None, **_: Any) -> tuple[dict[str, Any], list[EventPlan]]: return { "execution_generation": execution_generation, "reason": _terminal_text(result, field="reason", fallback="member_unavailable"), @@ -987,23 +865,14 @@ def _deferred_effects( _TERMINAL_EFFECTS = { - "settled": _settled_effects, - "failed": _failed_effects, - "cancelled": _cancelled_effects, + "settled": _settled_effects, "failed": _failed_effects, "cancelled": _cancelled_effects, "deferred": _deferred_effects, } def plan_publication( - room_value: Any, - events: Sequence[Mapping[str, Any]], - task: DiscussionTaskPlan, - *, - status: TerminalKind, - result: Any = None, - execution_generation: int | None = None, - local_profiles: Iterable[str], -) -> PublicationPlan: + room_value: Any, events: Sequence[Mapping[str, Any]], task: DiscussionTaskPlan, *, status: TerminalKind, + result: Any = None, execution_generation: int | None = None, local_profiles: Iterable[str]) -> PublicationPlan: """Plan idempotent room effects for one terminal driver task. A newer user event in the same thread supersedes a late result: the task @@ -1020,43 +889,26 @@ def plan_publication( raise DiscussionValidationError("invalid terminal publication status") if status == "deferred": message = "deferred publication requires an execution generation" - common.positive_int(execution_generation, error=DiscussionValidationError, message=message) + _bounded_int(execution_generation, message=message, low=1) newer_same_thread = any( event.kind == "message.user" and event.seq > task.seen_through_seq and event.payload.get("thread_id") == task.identity.thread_id - for event in validated - ) - effective_status: TerminalKind = ( - "cancelled" if newer_same_thread and status != "deferred" else status - ) + for event in validated) + effective_status: TerminalKind = ("cancelled" if newer_same_thread and status != "deferred" else status) digest = task.identity.task_id.removeprefix("dtask:") terminal_event_id = ( f"ddeferred:{digest}:g{execution_generation}" - if effective_status == "deferred" else f"dterminal:{digest}" - ) - common_payload = { - "discussion_event_id": task.discussion_event_id, - "member_id": task.member.member_id, - "member_index": task.member_index, - "round_index": task.round_index, - "seen_through_seq": task.seen_through_seq, - "task_id": task.identity.task_id, - "thread_id": task.identity.thread_id, - "turn_id": task.identity.turn_id, - } + if effective_status == "deferred" else f"dterminal:{digest}") + common_payload = {**_turn_coordinates(task), "seen_through_seq": task.seen_through_seq} extra, effects = _TERMINAL_EFFECTS[effective_status]( result, task=task, room=room, message_event_id=f"dmessage:{digest}", - newer_same_thread=newer_same_thread, execution_generation=execution_generation, - ) + newer_same_thread=newer_same_thread, execution_generation=execution_generation) terminal_kind = f"turn.{effective_status}" effects.append(EventPlan( event_id=terminal_event_id, kind=terminal_kind, actor={"kind": "gateway", "id": room.gateway_id}, payload={**common_payload, **extra}, - authority_gateway_id=room.gateway_id, authority_epoch=room.authority_epoch, - )) - return PublicationPlan( - task_id=task.identity.task_id, terminal_kind=terminal_kind, events=tuple(effects) - ) + authority_gateway_id=room.gateway_id, authority_epoch=room.authority_epoch)) + return PublicationPlan(task_id=task.identity.task_id, terminal_kind=terminal_kind, events=tuple(effects)) diff --git a/gateway/hosted_room_driver.py b/gateway/hosted_room_driver.py index 47b822754c..89533d2ae5 100644 --- a/gateway/hosted_room_driver.py +++ b/gateway/hosted_room_driver.py @@ -14,12 +14,12 @@ import json import math import sqlite3 from dataclasses import dataclass +from functools import partial from pathlib import Path from typing import Any, Callable, Literal, get_args from gateway.hosted_rooms_common import ( - canonical_json, identifier, non_negative_int, positive_int, table_columns, transaction, -) + bounded_int, canonical_json, compact_json, connect, identifier, table_columns, text, transaction) Clock = Callable[[], float] TaskStatus = Literal["queued", "running", "settled", "failed", "cancelled", "indeterminate", "deferred", "stopping"] @@ -38,16 +38,12 @@ _TASK_PAYLOAD_REQUIRED_FIELDS = frozenset({"target_profile", "prompt", "source_e _TASK_PAYLOAD_OPTIONAL_FIELDS = frozenset({"target_member_id"}) _LEASE_COLUMNS = frozenset({ "room_id", "gateway_id", "authority_epoch", "process_generation", - "lease_generation", "expires_at", "acquired_at", "updated_at", "released_at", -}) + "lease_generation", "expires_at", "acquired_at", "updated_at", "released_at"}) _TASK_COLUMN_ORDER = ( - "room_id", "task_id", "thread_id", "turn_id", "source_event_seq", - "payload_json", "payload_digest", "status", "execution_generation", - "cancel_generation", "run_gateway_id", "run_process_generation", - "run_lease_generation", "cancel_id", "settlement_id", "settlement_status", - "result_json", "created_at", "updated_at", "started_at", "terminal_at", - "indeterminate_at", -) + "room_id", "task_id", "thread_id", "turn_id", "source_event_seq", "payload_json", "payload_digest", + "status", "execution_generation", "cancel_generation", "run_gateway_id", "run_process_generation", + "run_lease_generation", "cancel_id", "settlement_id", "settlement_status", "result_json", "created_at", + "updated_at", "started_at", "terminal_at", "indeterminate_at") _TASK_COLUMNS = frozenset(_TASK_COLUMN_ORDER) _TASK_ORDER = "ORDER BY source_event_seq, created_at, task_id" _SELECT_LEASE = "SELECT * FROM hosted_room_driver_leases WHERE room_id=?" @@ -78,79 +74,64 @@ _SETTLE_RUNNING_SQL = _generation_update(_SETTLE_SET, "running") + f" AND {_RUN_ _SETTLE_STOPPING_SQL = _generation_update(_SETTLE_SET, "stopping") _REQUEUE_RUNNING_SQL = _task_update( f"{_REQUEUE_SET}, started_at=NULL, updated_at=?", - f"status='running' AND {_GENERATION_FENCE} AND {_RUN_FENCE}", -) + f"status='running' AND {_GENERATION_FENCE} AND {_RUN_FENCE}") _CANCEL_QUEUED_SQL = _task_update(_CANCEL_SET, "status IN ('queued', 'deferred') AND cancel_generation=?") _BEGIN_STOP_SQL = _task_update( "status='stopping', cancel_generation=?, cancel_id=?, updated_at=?", - "status IN ('running', 'indeterminate') AND cancel_generation=?", -) + "status IN ('running', 'indeterminate') AND cancel_generation=?") _COMPLETE_STOP_SQL = _task_update( "status='cancelled', terminal_at=?, updated_at=?", - "status='stopping' AND cancel_id=? AND cancel_generation=?", -) + "status='stopping' AND cancel_id=? AND cancel_generation=?") # Lease-first recovery transitions: name -> (fenced status, SET clause, generation-guard stale message, # row stale message); the UPDATE is _generation_update(set_clause, status). _INDETERMINATE_STALE = "indeterminate task generation changed" _GENERATION_TRANSITIONS = { - "resolve": ( - "indeterminate", _SETTLE_SET, _INDETERMINATE_STALE, "indeterminate task changed during reconciliation", - ), + "resolve": ("indeterminate", _SETTLE_SET, _INDETERMINATE_STALE, "indeterminate task changed during reconciliation"), "resolve_cancel": ( "indeterminate", _CANCEL_SET, "indeterminate cancellation proof is stale", - "indeterminate cancellation proof lost its fence", - ), + "indeterminate cancellation proof lost its fence"), "requeue": ( "indeterminate", f"{_REQUEUE_SET}, started_at=NULL, indeterminate_at=NULL, updated_at=?", - _INDETERMINATE_STALE, "indeterminate task changed during requeue", - ), + _INDETERMINATE_STALE, "indeterminate task changed during requeue"), "defer": ( "indeterminate", "status='deferred', result_json=?, terminal_at=?, updated_at=?", - _INDETERMINATE_STALE, "indeterminate task changed during deferral", - ), + _INDETERMINATE_STALE, "indeterminate task changed during deferral"), "requeue_deferred": ( "deferred", f"{_REQUEUE_SET}, result_json=NULL, started_at=NULL, terminal_at=NULL, indeterminate_at=NULL, updated_at=?", - "deferred task generation changed", "deferred task changed during requeue", - ), -} + "deferred task generation changed", "deferred task changed during requeue")} -class DriverStateError(ValueError): - """Base class for invalid or conflicting driver-state operations.""" +class DriverStateError(ValueError): """Base class for invalid or conflicting driver-state operations.""" -class DriverValidationError(DriverStateError): - """Raised when an identifier, clock, TTL, or payload is invalid.""" +class DriverValidationError(DriverStateError): """Raised when an identifier, clock, TTL, or payload is invalid.""" -class RoomUnavailableError(DriverStateError): - """Raised when the hosted room does not exist or was disbanded.""" +class RoomUnavailableError(DriverStateError): """Raised when the hosted room does not exist or was disbanded.""" -class LeaseHeldError(DriverStateError): - """Raised when another unexpired driver generation owns the room.""" +class LeaseHeldError(DriverStateError): """Raised when another unexpired driver generation owns the room.""" -class StaleLeaseError(DriverStateError): - """Raised when a lease generation can no longer mutate room state.""" +class StaleLeaseError(DriverStateError): """Raised when a lease generation can no longer mutate room state.""" -class TaskConflictError(DriverStateError): - """Raised when an idempotency key is reused for different task state.""" +class TaskConflictError(DriverStateError): """Raised when an idempotency key is reused for different task state.""" -class StaleTaskError(DriverStateError): - """Raised when an obsolete task attempt or cancellation tries to commit.""" +class StaleTaskError(DriverStateError): """Raised when an obsolete task attempt or cancellation tries to commit.""" -class InvalidTaskTransitionError(DriverStateError): - """Raised when a requested task transition is not allowed.""" +class InvalidTaskTransitionError(DriverStateError): """Raised when a requested task transition is not allowed.""" -def _identifier(value: Any, *, label: str) -> str: - return identifier(value, label=label, error=DriverValidationError, max_chars=MAX_IDENTIFIER_CHARS) +_identifier = partial(identifier, error=DriverValidationError, max_chars=MAX_IDENTIFIER_CHARS) +_bounded_int = partial(bounded_int, error=DriverValidationError) +_canonical_json = partial( + canonical_json, error=DriverValidationError, label="result", max_bytes=MAX_RESULT_JSON_BYTES, + ensure_ascii=True) def _finite(compute: Callable[[], Any], message: str, *, positive: bool = False) -> float: @@ -180,12 +161,6 @@ def _expiry(now: float, ttl: float) -> float: return expires_at -def _canonical_json(value: Any) -> str: - return canonical_json( - value, error=DriverValidationError, label="result", max_bytes=MAX_RESULT_JSON_BYTES, ensure_ascii=True - ) - - def _task_payload(value: Any) -> tuple[dict[str, Any], str, str]: if not isinstance(value, dict): raise DriverValidationError("payload must be an object") @@ -196,21 +171,13 @@ def _task_payload(value: Any) -> tuple[dict[str, Any], str, str]: if missing: raise DriverValidationError(f"missing payload fields: {', '.join(sorted(missing))}") target_profile = _identifier(value["target_profile"], label="target_profile") - prompt = value["prompt"] - if not isinstance(prompt, str): - raise DriverValidationError("prompt must be a string") - if not prompt.strip(): - raise DriverValidationError("prompt must not be empty") - if len(prompt.encode("utf-8")) > MAX_PROMPT_BYTES: - raise DriverValidationError("prompt is too large") - source_event_seq = positive_int( - value["source_event_seq"], error=DriverValidationError, - message="source_event_seq must be a positive integer", - ) + prompt = text(value["prompt"], error=DriverValidationError, label="prompt", max_bytes=MAX_PROMPT_BYTES, strip=False) + source_event_seq = _bounded_int( + value["source_event_seq"], message="source_event_seq must be a positive integer", low=1) normalized = {"target_profile": target_profile, "prompt": prompt, "source_event_seq": source_event_seq} if "target_member_id" in value: normalized["target_member_id"] = _identifier(value["target_member_id"], label="target_member_id") - encoded = json.dumps(normalized, ensure_ascii=True, sort_keys=True, separators=(",", ":")) + encoded = compact_json(normalized) return normalized, encoded, hashlib.sha256(encoded.encode("utf-8")).hexdigest() @@ -267,8 +234,7 @@ def _create_task_table(conn: sqlite3.Connection, table: str = "hosted_room_drive settlement_id TEXT, settlement_status TEXT, result_json TEXT, created_at REAL NOT NULL, updated_at REAL NOT NULL, started_at REAL, terminal_at REAL, indeterminate_at REAL, PRIMARY KEY (room_id, task_id), UNIQUE (room_id, thread_id, turn_id), - FOREIGN KEY (room_id) REFERENCES hosted_rooms(room_id))""" - ) + FOREIGN KEY (room_id) REFERENCES hosted_rooms(room_id))""") def _initialize_schema(conn: sqlite3.Connection) -> None: @@ -291,8 +257,7 @@ def _validate_schema(conn: sqlite3.Connection) -> None: if lease_columns != _LEASE_COLUMNS or task_columns != _TASK_COLUMNS: raise DriverStateError( "unsupported unpublished hosted-room driver schema; " - "recreate the driver tables before starting the driver" - ) + "recreate the driver tables before starting the driver") for table in ("hosted_room_driver_leases", "hosted_room_driver_tasks"): foreign_keys = conn.execute(f"PRAGMA foreign_key_list({table})").fetchall() if not any(row[2] == "hosted_rooms" and row[3] == "room_id" and row[4] == "room_id" for row in foreign_keys): @@ -307,15 +272,13 @@ def _schema_objects_exist(conn: sqlite3.Connection) -> bool: if {row[0] for row in rows} != {"hosted_room_driver_leases", "hosted_room_driver_tasks"}: return False index = conn.execute( - "SELECT 1 FROM sqlite_master WHERE type='index' AND name='idx_hosted_room_driver_tasks_status'" - ).fetchone() + "SELECT 1 FROM sqlite_master WHERE type='index' AND name='idx_hosted_room_driver_tasks_status'").fetchone() return index is not None def _task_schema_supports_current_statuses(conn: sqlite3.Connection) -> bool: row = conn.execute( - "SELECT sql FROM sqlite_master WHERE type='table' AND name='hosted_room_driver_tasks'" - ).fetchone() + "SELECT sql FROM sqlite_master WHERE type='table' AND name='hosted_room_driver_tasks'").fetchone() sql = str(row[0] or "").lower() if row else "" return "'stopping'" in sql and "'deferred'" in sql @@ -326,40 +289,33 @@ def _migrate_task_status_constraint(conn: sqlite3.Connection) -> None: _create_task_table(conn, "hosted_room_driver_tasks_next") columns = ", ".join(_TASK_COLUMN_ORDER) conn.execute( - f"INSERT INTO hosted_room_driver_tasks_next ({columns}) SELECT {columns} FROM hosted_room_driver_tasks" - ) + f"INSERT INTO hosted_room_driver_tasks_next ({columns}) SELECT {columns} FROM hosted_room_driver_tasks") conn.execute("DROP TABLE hosted_room_driver_tasks") conn.execute("ALTER TABLE hosted_room_driver_tasks_next RENAME TO hosted_room_driver_tasks") conn.execute(_TASK_INDEX_SQL.format(if_not_exists="")) def _connect(db_path: Path | str) -> sqlite3.Connection: - from hermes_state import apply_wal_with_fallback + """Open the store; existing tables are validated (after the status-constraint migration when needed). - path = Path(db_path) - path.parent.mkdir(parents=True, exist_ok=True) - conn = sqlite3.connect(path, timeout=10) - conn.row_factory = sqlite3.Row - try: - apply_wal_with_fallback(conn, db_label="state.db (hosted_room_driver)") - conn.execute("PRAGMA foreign_keys=ON") - if _schema_objects_exist(conn): - if not _task_schema_supports_current_statuses(conn): - conn.execute("BEGIN IMMEDIATE") - _migrate_task_status_constraint(conn) - conn.commit() + The driver schema never shipped, so an incompatible draft fails closed in ``_validate_schema``. + """ + existing: list[bool] = [] + + def ready(conn: sqlite3.Connection) -> bool: + existing.append(_schema_objects_exist(conn)) + return existing[0] and _task_schema_supports_current_statuses(conn) + + def initialize(conn: sqlite3.Connection) -> None: + (_migrate_task_status_constraint if existing[0] else _initialize_schema)(conn) + + conn = connect(db_path, db_label="state.db (hosted_room_driver)", ready=ready, initialize=initialize) + if existing[0]: + try: _validate_schema(conn) - return conn - # Schema creation is one database-wide transaction. The driver schema - # has never shipped, so an incompatible draft schema fails closed - # instead of attempting a partial in-place migration. - conn.execute("BEGIN IMMEDIATE") - _initialize_schema(conn) - conn.commit() - except Exception: - conn.rollback() - conn.close() - raise + except Exception: + conn.close() + raise return conn @@ -369,17 +325,14 @@ def _transaction(db_path: Path | str): def _lease_from_row(row: sqlite3.Row | dict[str, Any], *, reclaimed: bool = False) -> DriverLease: return DriverLease( - room_id=row["room_id"], gateway_id=row["gateway_id"], - authority_epoch=int(row["authority_epoch"]), process_generation=row["process_generation"], - lease_generation=int(row["lease_generation"]), expires_at=float(row["expires_at"]), - reclaimed=reclaimed, - ) + room_id=row["room_id"], gateway_id=row["gateway_id"], authority_epoch=int(row["authority_epoch"]), + process_generation=row["process_generation"], lease_generation=int(row["lease_generation"]), + expires_at=float(row["expires_at"]), reclaimed=reclaimed) def _task_identity_from_row(row: sqlite3.Row) -> TaskIdentity: return TaskIdentity( - room_id=row["room_id"], task_id=row["task_id"], thread_id=row["thread_id"], turn_id=row["turn_id"] - ) + room_id=row["room_id"], task_id=row["task_id"], thread_id=row["thread_id"], turn_id=row["turn_id"]) def _optional(cast: Callable[[Any], Any]) -> Callable[[Any], Any]: @@ -401,8 +354,7 @@ def _task_from_row(row: sqlite3.Row, *, idempotent: bool = False) -> dict[str, A except (TypeError, json.JSONDecodeError, DriverValidationError) as exc: raise TaskConflictError("stored task payload is invalid") from exc if (encoded_payload, payload_digest, payload["source_event_seq"]) != ( - row["payload_json"], row["payload_digest"], int(row["source_event_seq"]) - ): + row["payload_json"], row["payload_digest"], int(row["source_event_seq"])): raise TaskConflictError("stored task payload failed its integrity check") task: dict[str, Any] = {"identity": _task_identity_from_row(row), "payload": payload} for column in _TASK_COLUMN_ORDER[_TASK_COLUMN_ORDER.index("payload_digest"):]: @@ -430,8 +382,7 @@ def _load_active_room(conn: sqlite3.Connection, room_id: str) -> sqlite3.Row: try: row = conn.execute( "SELECT room_id, authority_gateway_id, authority_epoch, disbanded_at FROM hosted_rooms WHERE room_id=?", - (room_id,), - ).fetchone() + (room_id,)).fetchone() except sqlite3.OperationalError as exc: if "no such table" in str(exc).lower(): raise RoomUnavailableError("hosted room does not exist") from exc @@ -444,8 +395,7 @@ def _load_active_room(conn: sqlite3.Connection, room_id: str) -> sqlite3.Row: def _require_room_authority( - conn: sqlite3.Connection, *, room_id: str, gateway_id: str, authority_epoch: int -) -> sqlite3.Row: + conn: sqlite3.Connection, *, room_id: str, gateway_id: str, authority_epoch: int) -> sqlite3.Row: room = _load_active_room(conn, room_id) if room["authority_gateway_id"] != gateway_id or int(room["authority_epoch"]) != authority_epoch: raise StaleLeaseError("hosted room authority changed") @@ -454,8 +404,7 @@ def _require_room_authority( def _require_lease_authority(conn: sqlite3.Connection, lease: DriverLease) -> sqlite3.Row: return _require_room_authority( - conn, room_id=lease.room_id, gateway_id=lease.gateway_id, authority_epoch=lease.authority_epoch - ) + conn, room_id=lease.room_id, gateway_id=lease.gateway_id, authority_epoch=lease.authority_epoch) def _lease_row_matches(row: sqlite3.Row | None, lease: DriverLease) -> bool: @@ -485,8 +434,7 @@ def _cancel_generation(value: int) -> int: def _expected_generations( - lease: DriverLease, identity: TaskIdentity, execution_generation: int, cancel_generation: int -) -> None: + lease: DriverLease, identity: TaskIdentity, execution_generation: int, cancel_generation: int) -> None: _check_same_room(lease, identity) if not isinstance(execution_generation, int) or execution_generation < 1: raise DriverValidationError("expected_execution_generation must be a positive integer") @@ -515,14 +463,12 @@ def _settlement_replay(settlement_id: str, status: str, result_json: str) -> Cal def _cancel_replay(cancel_id: str, status: str = "cancelled") -> Callable[[sqlite3.Row], Any]: """Replay predicate: same cancel_id already committed in ``status``.""" return lambda row: ( - _task_from_row(row, idempotent=True) if row["status"] == status and row["cancel_id"] == cancel_id else None - ) + _task_from_row(row, idempotent=True) if row["status"] == status and row["cancel_id"] == cancel_id else None) def _generations_match(row: sqlite3.Row, status: str, execution_generation: int, cancel_generation: int) -> bool: return (row["status"], int(row["execution_generation"]), int(row["cancel_generation"])) == ( - status, execution_generation, cancel_generation - ) + status, execution_generation, cancel_generation) def _require_cancel_generation(row: sqlite3.Row, expected_cancel_generation: int) -> None: @@ -534,14 +480,12 @@ def _transition( db_path: Path | str, identity: TaskIdentity, *, sql: str, set_params: tuple[Any, ...], fence_params: tuple[Any, ...], stale: str, now: float, lease: DriverLease | None = None, lease_first: bool = True, replay: Callable[[sqlite3.Row], dict[str, Any] | None] | None = None, - guard: Callable[[sqlite3.Row], None] | None = None, -) -> dict[str, Any]: + guard: Callable[[sqlite3.Row], None] | None = None) -> dict[str, Any]: """Run one fenced task transition: load -> idempotent replay -> lease/fence guard -> UPDATE. - ``sql`` binds ``(*set_params, room_id, task_id, *fence_params)`` and must hit exactly one row - or ``stale`` is raised. ``lease_first`` puts the active-lease check before the row load - (recovery paths) instead of after the replay (settlement paths: an identical replay still - succeeds after the lease moved on). + ``sql`` binds ``(*set_params, room_id, task_id, *fence_params)`` and must hit exactly one row or + ``stale`` is raised. ``lease_first`` checks the lease before the row load (recovery paths) instead + of after the replay (settlement paths: an identical replay still succeeds after the lease moved on). """ params = (*set_params, identity.room_id, identity.task_id, *fence_params) with _transaction(db_path) as conn: @@ -564,8 +508,7 @@ def _transition( def _generation_transition( db_path: Path | str, identity: TaskIdentity, lease: DriverLease, name: str, execution_generation: int, cancel_generation: int, *, now: float, set_params: tuple[Any, ...], - replay: Callable[[sqlite3.Row], Any] | None = None, -) -> dict[str, Any]: + replay: Callable[[sqlite3.Row], Any] | None = None) -> dict[str, Any]: """Lease-first transition from ``_GENERATION_TRANSITIONS`` fenced on status + both generations.""" status, set_clause, generation_stale, stale = _GENERATION_TRANSITIONS[name] @@ -576,18 +519,15 @@ def _generation_transition( return _transition( db_path, identity, lease=lease, now=now, replay=replay, guard=guard, sql=_generation_update(set_clause, status), set_params=set_params, - fence_params=(execution_generation, cancel_generation), stale=stale, - ) + fence_params=(execution_generation, cancel_generation), stale=stale) def _run_fence_transition( db_path: Path | str, attempt: TaskAttempt, *, guard_stale: str, - lease_generation: Callable[[Any], int] = int, **transition: Any, -) -> dict[str, Any]: + lease_generation: Callable[[Any], int] = int, **transition: Any) -> dict[str, Any]: """Transition fenced on this attempt's running generation under its exact lease (row guard + SQL fence). - ``lease_generation`` casts the stored run_lease_generation: ``int`` raises on NULL, - ``int(v or 0)`` reads it as generation 0. + ``lease_generation`` casts the stored run_lease_generation: ``int`` raises on NULL, ``int(v or 0)`` reads 0. """ lease = attempt.lease @@ -596,30 +536,24 @@ def _run_fence_transition( _generations_match(row, "running", attempt.execution_generation, attempt.cancel_generation) and row["run_gateway_id"] == lease.gateway_id and row["run_process_generation"] == lease.process_generation - and lease_generation(row["run_lease_generation"]) == lease.lease_generation - ): + and lease_generation(row["run_lease_generation"]) == lease.lease_generation): raise StaleTaskError(guard_stale) return _transition( db_path, attempt.identity, lease=lease, guard=guard, fence_params=( attempt.execution_generation, attempt.cancel_generation, - lease.gateway_id, lease.process_generation, lease.lease_generation, - ), - **transition, - ) + lease.gateway_id, lease.process_generation, lease.lease_generation), + **transition) def acquire_lease( db_path: Path | str, *, room_id: Any, gateway_id: Any, authority_epoch: Any, process_generation: Any, - ttl_seconds: Any, clock: Clock, -) -> DriverLease: + ttl_seconds: Any, clock: Clock) -> DriverLease: """Acquire an empty or expired room lease with a monotonic generation.""" room_id = _identifier(room_id, label="room_id") gateway_id = _identifier(gateway_id, label="gateway_id") - authority_epoch = positive_int( - authority_epoch, error=DriverValidationError, message="authority_epoch must be a positive integer" - ) + authority_epoch = _bounded_int(authority_epoch, message="authority_epoch must be a positive integer", low=1) process_generation = _identifier(process_generation, label="process_generation") ttl_seconds = _ttl(ttl_seconds) now = _timestamp(clock) @@ -634,8 +568,7 @@ def acquire_lease( room_id, gateway_id, authority_epoch, process_generation, lease_generation, expires_at, acquired_at, updated_at, released_at ) VALUES (?, ?, ?, ?, 1, ?, ?, ?, NULL)""", - (room_id, gateway_id, authority_epoch, process_generation, expires_at, now, now), - ) + (room_id, gateway_id, authority_epoch, process_generation, expires_at, now, now)) return _lease_from_row(conn.execute(_SELECT_LEASE, (room_id,)).fetchone()) same_authority = row["gateway_id"] == gateway_id and int(row["authority_epoch"]) == authority_epoch @@ -644,8 +577,7 @@ def acquire_lease( renewed_expiry = max(float(row["expires_at"]), expires_at) conn.execute( "UPDATE hosted_room_driver_leases SET expires_at=?, updated_at=? WHERE room_id=? AND lease_generation=?", - (renewed_expiry, now, room_id, int(row["lease_generation"])), - ) + (renewed_expiry, now, room_id, int(row["lease_generation"]))) return _lease_from_row({**dict(row), "expires_at": renewed_expiry}) if same_authority and live: raise LeaseHeldError("room driver lease is held by another generation") @@ -658,9 +590,7 @@ def acquire_lease( gateway_id != ? OR authority_epoch != ? OR released_at IS NOT NULL OR expires_at <= ?)""", ( gateway_id, authority_epoch, process_generation, expires_at, now, now, - room_id, int(row["lease_generation"]), gateway_id, authority_epoch, now, - ), - ) + room_id, int(row["lease_generation"]), gateway_id, authority_epoch, now)) if updated.rowcount != 1: raise LeaseHeldError("room driver lease changed during acquisition") return _lease_from_row(conn.execute(_SELECT_LEASE, (room_id,)).fetchone(), reclaimed=True) @@ -678,8 +608,7 @@ def renew_lease(db_path: Path | str, lease: DriverLease, *, ttl_seconds: Any, cl """UPDATE hosted_room_driver_leases SET expires_at=?, updated_at=? WHERE room_id=? AND gateway_id=? AND process_generation=? AND lease_generation=? AND released_at IS NULL AND expires_at > ?""", - (expires_at, now, lease.room_id, lease.gateway_id, lease.process_generation, lease.lease_generation, now), - ) + (expires_at, now, lease.room_id, lease.gateway_id, lease.process_generation, lease.lease_generation, now)) if updated.rowcount != 1: raise StaleLeaseError("driver lease changed during renewal") return dataclasses.replace(lease, expires_at=expires_at, reclaimed=False) @@ -705,8 +634,7 @@ def release_lease(db_path: Path | str, lease: DriverLease, *, clock: Clock) -> d conn.execute( """UPDATE hosted_room_driver_leases SET expires_at=?, updated_at=?, released_at=? WHERE room_id=? AND lease_generation=?""", - (now, now, now, lease.room_id, lease.lease_generation), - ) + (now, now, now, lease.room_id, lease.lease_generation)) current = {**dict(row), "expires_at": now, "updated_at": now, "released_at": now} return {"lease": _lease_from_row(current), "idempotent": False} @@ -726,8 +654,7 @@ def admit_task(db_path: Path | str, identity: TaskIdentity, *, payload: Any, clo return _task_from_row(existing, idempotent=True) turn = conn.execute( "SELECT * FROM hosted_room_driver_tasks WHERE room_id=? AND thread_id=? AND turn_id=?", - (identity.room_id, identity.thread_id, identity.turn_id), - ).fetchone() + (identity.room_id, identity.thread_id, identity.turn_id)).fetchone() if turn is not None: raise TaskConflictError("thread_id and turn_id are already bound to a task") conn.execute( @@ -737,9 +664,7 @@ def admit_task(db_path: Path | str, identity: TaskIdentity, *, payload: Any, clo ) VALUES (?, ?, ?, ?, ?, ?, ?, 'queued', 0, 0, ?, ?)""", ( identity.room_id, identity.task_id, identity.thread_id, identity.turn_id, - normalized_payload["source_event_seq"], payload_json, payload_digest, now, now, - ), - ) + normalized_payload["source_event_seq"], payload_json, payload_digest, now, now)) return _task_from_row(_load_task(conn, identity)) @@ -759,14 +684,12 @@ def start_task( unresolved = conn.execute( f"""SELECT task_id, status FROM hosted_room_driver_tasks WHERE room_id=? AND status IN ('running', 'indeterminate', 'stopping') {_TASK_ORDER} LIMIT 1""", - (identity.room_id,), - ).fetchone() + (identity.room_id,)).fetchone() if unresolved is not None: raise InvalidTaskTransitionError("room recovery must resolve the prior task before starting new work") next_queued = conn.execute( f"SELECT task_id FROM hosted_room_driver_tasks WHERE room_id=? AND status='queued' {_TASK_ORDER} LIMIT 1", - (identity.room_id,), - ).fetchone() + (identity.room_id,)).fetchone() if next_queued is None or next_queued["task_id"] != identity.task_id: raise InvalidTaskTransitionError("task is not next in the hosted room event order") execution_generation = int(row["execution_generation"]) + 1 @@ -777,15 +700,12 @@ def start_task( WHERE room_id=? AND task_id=? AND status='queued' AND cancel_generation=?""", ( execution_generation, lease.gateway_id, lease.process_generation, lease.lease_generation, now, now, - identity.room_id, identity.task_id, expected_cancel_generation, - ), - ) + identity.room_id, identity.task_id, expected_cancel_generation)) if updated.rowcount != 1: raise StaleTaskError("task changed during start") return TaskAttempt( identity=identity, lease=lease, - execution_generation=execution_generation, cancel_generation=expected_cancel_generation, - ) + execution_generation=execution_generation, cancel_generation=expected_cancel_generation) def settle_task( @@ -799,8 +719,7 @@ def settle_task( db_path, attempt, guard_stale="task attempt is stale or cancelled", lease_first=False, now=now, replay=_settlement_replay(settlement_id, status, result_json), sql=_SETTLE_RUNNING_SQL, set_params=(status, settlement_id, status, result_json, now, now), - stale="task changed during settlement", - ) + stale="task changed during settlement") def settle_stopping_task( @@ -818,8 +737,7 @@ def settle_stopping_task( replay=_settlement_replay(settlement_id, status, result_json), sql=_SETTLE_STOPPING_SQL, set_params=(status, settlement_id, status, result_json, now, now), fence_params=(expected_execution_generation, expected_cancel_generation), - stale="task completion lost the stop race", - ) + stale="task completion lost the stop race") def resolve_indeterminate_task( @@ -834,41 +752,35 @@ def resolve_indeterminate_task( return _generation_transition( db_path, identity, lease, "resolve", expected_execution_generation, expected_cancel_generation, now=now, replay=_settlement_replay(settlement_id, status, result_json), - set_params=(status, settlement_id, status, result_json, now, now), - ) + set_params=(status, settlement_id, status, result_json, now, now)) def resolve_indeterminate_cancellation( db_path: Path | str, identity: TaskIdentity, lease: DriverLease, *, expected_execution_generation: int, - expected_cancel_generation: int, cancel_id: Any, clock: Clock, -) -> dict[str, Any]: + expected_cancel_generation: int, cancel_id: Any, clock: Clock) -> dict[str, Any]: """Commit a verified terminal cancellation for an uncertain attempt.""" _expected_generations(lease, identity, expected_execution_generation, expected_cancel_generation) cancel_id = _identifier(cancel_id, label="cancel_id") now = _timestamp(clock) return _generation_transition( db_path, identity, lease, "resolve_cancel", expected_execution_generation, expected_cancel_generation, - now=now, replay=_cancel_replay(cancel_id), set_params=(expected_cancel_generation + 1, cancel_id, now, now), - ) + now=now, replay=_cancel_replay(cancel_id), set_params=(expected_cancel_generation + 1, cancel_id, now, now)) def requeue_indeterminate_task( db_path: Path | str, identity: TaskIdentity, lease: DriverLease, *, expected_execution_generation: int, - expected_cancel_generation: int, clock: Clock, -) -> dict[str, Any]: + expected_cancel_generation: int, clock: Clock) -> dict[str, Any]: """Explicitly retry uncertain work after an operator accepts at-least-once risk.""" _expected_generations(lease, identity, expected_execution_generation, expected_cancel_generation) now = _timestamp(clock) return _generation_transition( db_path, identity, lease, "requeue", expected_execution_generation, expected_cancel_generation, now=now, - set_params=(now,), - ) + set_params=(now,)) def defer_indeterminate_task( db_path: Path | str, identity: TaskIdentity, lease: DriverLease, *, expected_execution_generation: int, - expected_cancel_generation: int, reason: Any, clock: Clock, -) -> dict[str, Any]: + expected_cancel_generation: int, reason: Any, clock: Clock) -> dict[str, Any]: """Fence one uncertain attempt and release later room work.""" _expected_generations(lease, identity, expected_execution_generation, expected_cancel_generation) reason = _identifier(reason, label="defer_reason") @@ -881,21 +793,18 @@ def defer_indeterminate_task( return _generation_transition( db_path, identity, lease, "defer", expected_execution_generation, expected_cancel_generation, now=now, - replay=replay, set_params=(result_json, now, now), - ) + replay=replay, set_params=(result_json, now, now)) def requeue_deferred_task( db_path: Path | str, identity: TaskIdentity, lease: DriverLease, *, expected_execution_generation: int, - expected_cancel_generation: int, clock: Clock, -) -> dict[str, Any]: + expected_cancel_generation: int, clock: Clock) -> dict[str, Any]: """Explicitly retry a fenced deferred turn under a new generation.""" _expected_generations(lease, identity, expected_execution_generation, expected_cancel_generation) now = _timestamp(clock) return _generation_transition( db_path, identity, lease, "requeue_deferred", expected_execution_generation, expected_cancel_generation, - now=now, set_params=(now,), - ) + now=now, set_params=(now,)) def requeue_not_admitted_task(db_path: Path | str, attempt: TaskAttempt, *, clock: Clock) -> dict[str, Any]: @@ -908,15 +817,13 @@ def requeue_not_admitted_task(db_path: Path | str, attempt: TaskAttempt, *, cloc _generations_match(row, "queued", attempt.execution_generation, attempt.cancel_generation) and row["run_gateway_id"] is None and row["run_process_generation"] is None - and row["run_lease_generation"] is None - ) + and row["run_lease_generation"] is None) return _task_from_row(row, idempotent=True) if requeued else None return _run_fence_transition( db_path, attempt, guard_stale="not-admitted task attempt lost its fence", lease_generation=lambda value: int(value or 0), now=now, replay=replay, - sql=_REQUEUE_RUNNING_SQL, set_params=(now,), stale="not-admitted task changed during requeue", - ) + sql=_REQUEUE_RUNNING_SQL, set_params=(now,), stale="not-admitted task changed during requeue") def cancel_task( @@ -937,8 +844,7 @@ def cancel_task( return _transition( db_path, identity, now=now, replay=_cancel_replay(cancel_id), guard=guard, sql=_CANCEL_QUEUED_SQL, set_params=(expected_cancel_generation + 1, cancel_id, now, now), fence_params=(expected_cancel_generation,), - stale="task changed during cancellation", - ) + stale="task changed during cancellation") def begin_task_cancel( @@ -957,8 +863,7 @@ def begin_task_cancel( return _transition( db_path, identity, now=now, replay=_cancel_replay(cancel_id, "stopping"), guard=guard, sql=_BEGIN_STOP_SQL, set_params=(expected_cancel_generation + 1, cancel_id, now), fence_params=(expected_cancel_generation,), - stale="task changed during stop request", - ) + stale="task changed during stop request") def complete_task_cancel( @@ -970,15 +875,13 @@ def complete_task_cancel( def guard(row: sqlite3.Row) -> None: if (row["status"], row["cancel_id"], int(row["cancel_generation"])) != ( - "stopping", cancel_id, expected_cancel_generation - ): + "stopping", cancel_id, expected_cancel_generation): raise StaleTaskError("task stop acknowledgement is stale") return _transition( db_path, identity, now=now, replay=_cancel_replay(cancel_id), guard=guard, sql=_COMPLETE_STOP_SQL, set_params=(now, now), fence_params=(cancel_id, expected_cancel_generation), - stale="task changed during stop acknowledgement", - ) + stale="task changed during stop acknowledgement") def recover_room(db_path: Path | str, lease: DriverLease, *, clock: Clock) -> dict[str, list[TaskIdentity]]: @@ -989,18 +892,15 @@ def recover_room(db_path: Path | str, lease: DriverLease, *, clock: Clock) -> di with _transaction(db_path) as conn: _require_active_lease(conn, lease, now=now) stale_rows = conn.execute( - f"SELECT * FROM hosted_room_driver_tasks WHERE {foreign_running} {_TASK_ORDER}", fence - ).fetchall() + f"SELECT * FROM hosted_room_driver_tasks WHERE {foreign_running} {_TASK_ORDER}", fence).fetchall() if stale_rows: conn.execute( f"""UPDATE hosted_room_driver_tasks SET status='indeterminate', indeterminate_at=?, updated_at=? WHERE {foreign_running}""", - (now, now, *fence), - ) + (now, now, *fence)) return { status: [_task_identity_from_row(row) for row in _tasks_in_order(conn, lease.room_id, status)] - for status in ("queued", "indeterminate") - } + for status in ("queued", "indeterminate")} def _read(db_path: Path | str, query: Callable[[sqlite3.Connection], Any]) -> Any: @@ -1026,19 +926,17 @@ def list_tasks(db_path: Path | str, *, room_id: Any, status: TaskStatus | None = def prune_published_terminal_tasks( db_path: Path | str, *, room_id: Any, clock: Clock, - retention_seconds: float = TERMINAL_TASK_RETENTION_SECONDS, retain: int = MAX_RETAINED_TERMINAL_TASKS, -) -> int: + retention_seconds: float = TERMINAL_TASK_RETENTION_SECONDS, retain: int = MAX_RETAINED_TERMINAL_TASKS) -> int: """Bound execution rows after outcomes are durable in the room log.""" room_id = _identifier(room_id, label="room_id") now = _timestamp(clock) if retention_seconds <= 0: raise DriverValidationError("retention_seconds must be positive") - non_negative_int(retain, error=DriverValidationError, message="retain must be a non-negative integer") + _bounded_int(retain, message="retain must be a non-negative integer") with _transaction(db_path) as conn: publications = conn.execute( - "SELECT 1 FROM sqlite_master WHERE type='table' AND name='hosted_room_policy_publications'" - ).fetchone() + "SELECT 1 FROM sqlite_master WHERE type='table' AND name='hosted_room_policy_publications'").fetchone() if publications is None: return 0 rows = conn.execute( @@ -1047,9 +945,7 @@ def prune_published_terminal_tasks( AND EXISTS (SELECT 1 FROM hosted_room_policy_publications p WHERE p.room_id=t.room_id AND p.task_id=t.task_id AND p.kind IN ('turn.settled', 'turn.failed', 'turn.cancelled')) - ORDER BY t.terminal_at DESC, t.task_id ASC""", - (room_id,), - ).fetchall() + ORDER BY t.terminal_at DESC, t.task_id ASC""", (room_id,)).fetchall() cutoff = now - float(retention_seconds) candidates = [ str(row["task_id"]) @@ -1061,6 +957,5 @@ def prune_published_terminal_tasks( placeholders = ",".join("?" for _ in candidates) deleted = conn.execute( f"DELETE FROM hosted_room_driver_tasks WHERE room_id=? AND task_id IN ({placeholders})", - (room_id, *candidates), - ) + (room_id, *candidates)) return max(0, int(deleted.rowcount)) diff --git a/gateway/hosted_room_execution_policy.py b/gateway/hosted_room_execution_policy.py index 41da1e08cf..6bc87f6a86 100644 --- a/gateway/hosted_room_execution_policy.py +++ b/gateway/hosted_room_execution_policy.py @@ -14,13 +14,10 @@ POLICY_VERSION = 1 MAX_POLICY_TOOLSETS = 128 MAX_POLICY_ITERATIONS = (1 << 53) - 1 _IDENTIFIER_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]*$") -_POLICY_FIELDS = { - "version", "target_profile", "enabled_toolsets", "approval_mode", "max_iterations", "policy_digest", -} +_POLICY_FIELDS = {"version", "target_profile", "enabled_toolsets", "approval_mode", "max_iterations", "policy_digest"} -class RoomExecutionPolicyError(ValueError): - """A RoomLink execution policy is malformed or no longer current.""" +class RoomExecutionPolicyError(ValueError): """A RoomLink execution policy is malformed or no longer current.""" def _policy_digest(unsigned: Mapping[str, Any]) -> str: @@ -67,15 +64,11 @@ class RoomExecutionPolicy: if ( isinstance(max_iterations, bool) or not isinstance(max_iterations, int) - or not 1 <= max_iterations <= MAX_POLICY_ITERATIONS - ): + or not 1 <= max_iterations <= MAX_POLICY_ITERATIONS): raise RoomExecutionPolicyError("max_iterations is invalid") unsigned = { - "version": POLICY_VERSION, - "target_profile": target_profile, - "enabled_toolsets": list(toolsets), - "approval_mode": approval_mode, - "max_iterations": max_iterations, + "version": POLICY_VERSION, "target_profile": target_profile, "enabled_toolsets": list(toolsets), + "approval_mode": approval_mode, "max_iterations": max_iterations, } supplied = str(value["policy_digest"] or "").strip().lower() if supplied != _policy_digest(unsigned): @@ -103,12 +96,9 @@ def execution_policy_mapping(*, target_profile: str, config: Mapping[str, Any] | agent = config.get("agent") if isinstance(config.get("agent"), Mapping) else {} approvals = config.get("approvals") if isinstance(config.get("approvals"), Mapping) else {} unsigned = { - "version": POLICY_VERSION, - "target_profile": _identifier(target_profile, field="target_profile"), + "version": POLICY_VERSION, "target_profile": _identifier(target_profile, field="target_profile"), "enabled_toolsets": toolsets, - "approval_mode": ( - "off" if _YOLO_MODE_FROZEN else _normalize_approval_mode(approvals.get("mode", "manual")) - ), + "approval_mode": ("off" if _YOLO_MODE_FROZEN else _normalize_approval_mode(approvals.get("mode", "manual"))), "max_iterations": min(resolve_turn_limit(agent.get("max_turns")), MAX_POLICY_ITERATIONS), } value = {**unsigned, "policy_digest": _policy_digest(unsigned)} @@ -131,12 +121,7 @@ def current_room_execution_policy() -> RoomExecutionPolicy | None: __all__ = [ - "MAX_POLICY_ITERATIONS", - "POLICY_VERSION", - "RoomExecutionPolicy", - "RoomExecutionPolicyError", - "bind_room_execution_policy", - "current_room_execution_policy", - "execution_policy_mapping", + "MAX_POLICY_ITERATIONS", "POLICY_VERSION", "RoomExecutionPolicy", "RoomExecutionPolicyError", + "bind_room_execution_policy", "current_room_execution_policy", "execution_policy_mapping", "reset_room_execution_policy", ] diff --git a/gateway/hosted_room_links.py b/gateway/hosted_room_links.py index 44363cfda3..7658b6be9c 100644 --- a/gateway/hosted_room_links.py +++ b/gateway/hosted_room_links.py @@ -12,6 +12,7 @@ import json import os import time from dataclasses import dataclass, field +from functools import partial from pathlib import Path from typing import Any, Mapping @@ -20,23 +21,21 @@ from gateway.hosted_room_peer import ( GatewayRoomCatalog, HostedRoomPeerError, TransportSecurity, - validate_room_link_url, -) + validate_room_link_url) +from gateway.hosted_rooms_common import compact_json, exact_fields MAX_LINKS = 512 MAX_GRANT_CHARS = 16 * 1024 _LEGACY_FIELDS = { "room_id", "member_id", "target_url", "target_profile", "grant", "catalog", - "cancellation_scope_id", "trace_id", "updated_at", -} + "cancellation_scope_id", "trace_id", "updated_at"} _OPTIONAL_FIELDS = {"transport_security", "status"} # SQLite record columns that map 1:1 onto mapping fields, in record order # (``catalog_json`` is the serialized ``catalog``). _RECORD_FIELDS = ( "room_id", "member_id", "target_url", "target_profile", "grant", "cancellation_scope_id", - "trace_id", "transport_security", "status", "updated_at", -) + "trace_id", "transport_security", "status", "updated_at") _STATUSES = {"ready", "unavailable", "needs_reauthorization"} @@ -56,8 +55,7 @@ class StoredRoomLink: @classmethod def from_mapping(cls, value: Mapping[str, Any]) -> "StoredRoomLink": - if set(value) - _LEGACY_FIELDS - _OPTIONAL_FIELDS or not _LEGACY_FIELDS.issubset(value): - raise HostedRoomPeerError("stored room link fields are invalid") + _link_fields(value) room_id = _short_string(value["room_id"], "room_id") member_id = _short_string(value["member_id"], "member_id") target_profile = _short_string(value["target_profile"], "target_profile") @@ -85,8 +83,7 @@ class StoredRoomLink: trace_id=_short_string(value["trace_id"], "trace_id"), transport_security=transport_security, # type: ignore[arg-type] status=status, - updated_at=updated_at, - ) + updated_at=updated_at) @classmethod def from_record(cls, value: Mapping[str, Any]) -> "StoredRoomLink": @@ -101,10 +98,16 @@ class StoredRoomLink: def as_record(self) -> dict[str, Any]: record = {name: getattr(self, name) for name in _RECORD_FIELDS} - record["catalog_json"] = json.dumps(self.catalog_mapping(), sort_keys=True, separators=(",", ":")) + record["catalog_json"] = compact_json(self.catalog_mapping(), ensure_ascii=False) return record +_link_fields = partial( + exact_fields, label="stored room link", required=_LEGACY_FIELDS, optional=_OPTIONAL_FIELDS, + error=HostedRoomPeerError, not_object="stored room link fields are invalid", + missing_fmt="stored room link fields are invalid", unknown_fmt="stored room link fields are invalid") + + def _short_string(value: Any, field: str) -> str: normalized = str(value or "").strip() if not normalized or len(normalized) > 256: @@ -149,32 +152,15 @@ def mark_room_link_status(db_path: Path | str, *, room_id: str, member_id: str, raise HostedRoomPeerError("stored room link status is invalid") return hosted_rooms.update_room_link_status( db_path, room_id=_short_string(room_id, "room_id"), member_id=_short_string(member_id, "member_id"), - status=status, - ) + status=status) def make_stored_link( - *, - room_id: str, - member_id: str, - target_url: str, - target_profile: str, - grant: str, - catalog: GatewayRoomCatalog, - cancellation_scope_id: str, - trace_id: str, -) -> StoredRoomLink: + *, room_id: str, member_id: str, target_url: str, target_profile: str, grant: str, + catalog: GatewayRoomCatalog, cancellation_scope_id: str, trace_id: str) -> StoredRoomLink: target_url, transport_security = validate_room_link_url(target_url) return StoredRoomLink.from_mapping({ - "room_id": room_id, - "member_id": member_id, - "target_url": target_url, - "target_profile": target_profile, - "grant": grant, - "catalog": catalog.as_mapping(), - "cancellation_scope_id": cancellation_scope_id, - "trace_id": trace_id, - "transport_security": transport_security, - "status": "ready", - "updated_at": time.time(), + "room_id": room_id, "member_id": member_id, "target_url": target_url, "target_profile": target_profile, + "grant": grant, "catalog": catalog.as_mapping(), "cancellation_scope_id": cancellation_scope_id, + "trace_id": trace_id, "transport_security": transport_security, "status": "ready", "updated_at": time.time(), }) diff --git a/gateway/hosted_room_peer.py b/gateway/hosted_room_peer.py index 31c302456f..bcb22f7d2b 100644 --- a/gateway/hosted_room_peer.py +++ b/gateway/hosted_room_peer.py @@ -16,7 +16,6 @@ import math import os import re import stat -import time import urllib.parse from dataclasses import asdict, dataclass from functools import lru_cache, partial @@ -24,7 +23,7 @@ from pathlib import Path from typing import Any, Callable, Iterable, Literal, Mapping from gateway.hosted_room_execution_policy import RoomExecutionPolicy, execution_policy_mapping -from gateway.hosted_rooms_common import exact_fields, identifier, positive_int +from gateway.hosted_rooms_common import bounded_int, clock, compact_json, exact_fields, identifier, text # v2 adds authority/member lineage to scoped grants and is deliberately not @@ -40,12 +39,10 @@ LinkMode = Literal["direct", "overlay", "relay", "pull", "desktop"] TransportSecurity = Literal["tls", "loopback"] -class HostedRoomPeerError(ValueError): - """Base error for malformed or unauthorized peer-room input.""" +class HostedRoomPeerError(ValueError): """Base error for malformed or unauthorized peer-room input.""" -class HostedRoomGrantError(HostedRoomPeerError): - """Raised when a room-scoped grant is invalid or expired.""" +class HostedRoomGrantError(HostedRoomPeerError): """Raised when a room-scoped grant is invalid or expired.""" _ROOM_GRANT_SECRET_FILE = ".room-link-grant-secret" @@ -104,10 +101,9 @@ def _gateway_room_grant_secret_for_home(home_value: str) -> bytes: def gateway_room_grant_secret(root: Path | str | None = None) -> bytes: """Load or atomically mint the gateway-only RoomLink signing secret. - API keys are bearer credentials known to clients and may be profile scoped; - they must never become grant-signing authority. This secret lives in the - installation root, is not exposed by configuration or capability RPCs, and - is shared only by the gateway processes that serve this installation. + API keys are client-known, possibly profile-scoped bearer credentials and must never become + grant-signing authority; this secret lives in the installation root, is never exposed by + config or capability RPCs, and is shared only by this installation's gateway processes. """ if root is None: from hermes_constants import get_hermes_home @@ -121,8 +117,7 @@ def gateway_room_grant_secret(root: Path | str | None = None) -> bytes: def derive_room_grant_secret(api_key: str) -> bytes: """Domain-separate room grants from the configured API key. - The API-server startup guard enforces the production key-strength policy; - this helper keeps a lower structural floor for isolated contract tests. + Production key strength is enforced by the API-server startup guard; this floor is for contract tests. """ if not isinstance(api_key, str) or len(api_key) < 8: raise HostedRoomGrantError("room grants require a strong gateway API key") @@ -132,12 +127,12 @@ def derive_room_grant_secret(api_key: str) -> bytes: def _identifier(value: Any, *, field: str) -> str: return identifier( value, label=field, error=HostedRoomPeerError, max_chars=256, pattern=_IDENTIFIER_RE, - invalid=f"{field} is invalid", - ) + invalid=f"{field} is invalid") def _positive_int(value: Any, *, field: str) -> int: - return positive_int(value, error=HostedRoomPeerError, message=f"{field} must be a positive integer") + return bounded_int( + value, error=HostedRoomPeerError, message=f"{field} must be a positive integer", low=1) def _digest(value: Any, *, field: str) -> str: @@ -148,12 +143,11 @@ def _digest(value: Any, *, field: str) -> str: _exact_fields = partial( exact_fields, error=HostedRoomPeerError, missing_fmt="{label} missing fields: {fields}", - unknown_fmt="{label} unknown fields: {fields}", -) + unknown_fmt="{label} unknown fields: {fields}") def _canonical_json(value: Mapping[str, Any]) -> bytes: - return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode("ascii") + return compact_json(value).encode("ascii") def _b64encode(value: bytes) -> str: @@ -198,8 +192,7 @@ def _parse_endpoint(endpoint: Any) -> tuple[str | None, str | None, TransportSec _CATALOG_FIELDS = { "installation_id", "protocol_versions", "link_modes", "persistent_process", "text", - "attachments", "execution_policy", "catalog_digest", -} + "attachments", "execution_policy", "catalog_digest"} @dataclass(frozen=True) @@ -239,9 +232,7 @@ class GatewayRoomCatalog: persistent_process=value["persistent_process"], text=value["text"], attachments=value["attachments"], execution_policy=policy, catalog_digest=_digest(value["catalog_digest"], field="catalog_digest"), - endpoint_url=endpoint_url, endpoint_reason=endpoint_reason, - transport_security=transport_security, - ) + endpoint_url=endpoint_url, endpoint_reason=endpoint_reason, transport_security=transport_security) unsigned = catalog.as_mapping() del unsigned["catalog_digest"] expected = hashlib.sha256(_canonical_json(unsigned)).hexdigest() @@ -255,8 +246,7 @@ class GatewayRoomCatalog: "installation_id": self.installation_id, "protocol_versions": list(self.protocol_versions), "link_modes": list(self.link_modes), "persistent_process": self.persistent_process, "text": self.text, "attachments": self.attachments, - "execution_policy": self.execution_policy.as_mapping(), "catalog_digest": self.catalog_digest, - } + "execution_policy": self.execution_policy.as_mapping(), "catalog_digest": self.catalog_digest} if self.endpoint_url is not None or self.endpoint_reason is not None: value["endpoint"] = self.endpoint_mapping() return value @@ -264,9 +254,7 @@ class GatewayRoomCatalog: def endpoint_mapping(self) -> dict[str, Any]: """Return the normalized self-advertised endpoint capability.""" if self.endpoint_url is not None: - return { - "available": True, "url": self.endpoint_url, "transport_security": self.transport_security, - } + return {"available": True, "url": self.endpoint_url, "transport_security": self.transport_security} return {"available": False, "reason": self.endpoint_reason or "not_configured"} @@ -274,18 +262,14 @@ def catalog_mapping( *, installation_id: str, protocol_versions: Iterable[int] = (PROTOCOL_VERSION,), link_modes: Iterable[LinkMode] = ("direct", "pull"), persistent_process: bool, text: bool = True, attachments: bool = False, endpoint: Mapping[str, Any] | None = None, target_profile: str | None = None, - execution_policy: Mapping[str, Any] | None = None, -) -> dict[str, Any]: + execution_policy: Mapping[str, Any] | None = None) -> dict[str, Any]: """Build a canonical catalog mapping with its digest.""" # A Desktop-managed gateway exits with the app: the caller's flag is only an # upper bound, so call sites that still pass ``True`` stay honest. persistent_process = bool(persistent_process and os.getenv("HERMES_DESKTOP") != "1") - profile = ( - str(target_profile or "").strip() or (os.getenv("HERMES_PROFILE") or "default").strip() or "default" - ) + profile = (str(target_profile or "").strip() or (os.getenv("HERMES_PROFILE") or "default").strip() or "default") checked_policy = RoomExecutionPolicy.from_mapping( - execution_policy or execution_policy_mapping(target_profile=profile) - ) + execution_policy or execution_policy_mapping(target_profile=profile)) # A RoomLink run is initiated by another installation. Process-wide YOLO # mode bypasses the scoped approval ContextVar, so rewriting the advertised # policy cannot make it safe: refuse until manual or smart approvals are on. @@ -293,16 +277,13 @@ def catalog_mapping( raise HostedRoomPeerError("remote room execution requires manual or smart approvals") value = { "installation_id": _identifier(installation_id, field="installation_id"), - "protocol_versions": sorted( - {_positive_int(item, field="protocol_version") for item in protocol_versions} - ), + "protocol_versions": sorted({_positive_int(item, field="protocol_version") for item in protocol_versions}), # Direct HTTPS/loopback is the only RoomLink transport implemented by # this backend slice. Do not advertise pull/relay placeholders. "link_modes": [mode for mode in dict.fromkeys(link_modes) if mode == "direct"], "persistent_process": persistent_process, "text": bool(text), "attachments": bool(attachments), "execution_policy": checked_policy.as_mapping(), - "endpoint": dict(local_room_link_endpoint() if endpoint is None else endpoint), - } + "endpoint": dict(local_room_link_endpoint() if endpoint is None else endpoint)} value["catalog_digest"] = hashlib.sha256(_canonical_json(value)).hexdigest() GatewayRoomCatalog.from_mapping(value) return value @@ -311,14 +292,12 @@ def catalog_mapping( def local_catalog_mapping( *, installation_id: str, protocol_versions: Iterable[int] = (PROTOCOL_VERSION,), link_modes: Iterable[LinkMode] = ("direct", "pull"), text: bool = True, attachments: bool = False, - target_profile: str | None = None, execution_policy: Mapping[str, Any] | None = None, -) -> dict[str, Any]: + target_profile: str | None = None, execution_policy: Mapping[str, Any] | None = None) -> dict[str, Any]: """Build the one truthful catalog advertised by this local process.""" return catalog_mapping( installation_id=installation_id, protocol_versions=protocol_versions, link_modes=link_modes, persistent_process=True, text=text, attachments=attachments, target_profile=target_profile, - execution_policy=execution_policy, - ) + execution_policy=execution_policy) def local_room_link_endpoint(value: Any | None = None) -> dict[str, Any]: @@ -369,11 +348,7 @@ def _configured_room_link_url() -> str | None: def validate_room_link_url(value: Any) -> tuple[str, TransportSecurity]: - """Validate a RoomLink endpoint and classify its transport protection. - - Plaintext HTTP is allowed only toward the local loopback interface; every - non-loopback endpoint must use HTTPS. - """ + """Validate a RoomLink endpoint and classify its transport: plaintext HTTP only toward loopback.""" raw = str(value or "").strip().rstrip("/") try: parsed = urllib.parse.urlsplit(raw) @@ -445,27 +420,23 @@ class HostedMemberDispatch: prompt = value["prompt"] if not isinstance(prompt, str) or not prompt.strip(): raise HostedRoomPeerError("prompt must be a non-empty string") - if len(prompt.encode("utf-8")) > MAX_PROMPT_BYTES: - raise HostedRoomPeerError("prompt is too large") + text(prompt, error=HostedRoomPeerError, label="prompt", max_bytes=MAX_PROMPT_BYTES, strip=False) expected_prompt_digest = hashlib.sha256(prompt.encode("utf-8")).hexdigest() prompt_digest = _digest(value["prompt_digest"], field="prompt_digest") if not hmac.compare_digest(expected_prompt_digest, prompt_digest): raise HostedRoomPeerError("prompt_digest does not match prompt") return cls( prompt=prompt, prompt_digest=prompt_digest, - **{name: check(value[name], field=name) for name, check in _DISPATCH_FIELDS.items()}, - ) + **{name: check(value[name], field=name) for name, check in _DISPATCH_FIELDS.items()}) # Grant scope fields, in issue-time validation order; each is validated with # the same checker as the matching dispatch field (``grant_id`` is an identifier). _GRANT_SCOPE = ( "grant_id", "room_id", "home_install_id", "authority_gateway_id", "authority_epoch", - "member_id", "target_install_id", "target_profile", -) + "member_id", "target_install_id", "target_profile") _GRANT_FIELDS = frozenset({ - "version", *_GRANT_SCOPE, "execution_policy_digest", "permissions", "issued_at", "expires_at", -}) + "version", *_GRANT_SCOPE, "execution_policy_digest", "permissions", "issued_at", "expires_at"}) _GRANT_REFRESH_FIELDS = _GRANT_FIELDS | {"status_expires_at"} _GRANT_PERMISSIONS = {"approve", "dispatch", "status", "stop"} MAX_DISPATCH_GRANT_TTL_SECONDS = 24 * 60 * 60 @@ -476,24 +447,20 @@ def issue_room_grant( secret: bytes, *, grant_id: str, room_id: str, home_install_id: str, authority_gateway_id: str, authority_epoch: int, member_id: str, target_install_id: str, target_profile: str, execution_policy_digest: str | None = None, - permissions: Iterable[str] = ("approve", "dispatch", "status", "stop"), - issued_at: float | None = None, ttl_seconds: float = 3600, status_ttl_seconds: float | None = None, - status_expires_at: float | None = None, -) -> str: + permissions: Iterable[str] = ("approve", "dispatch", "status", "stop"), issued_at: float | None = None, + ttl_seconds: float = 3600, status_ttl_seconds: float | None = None, status_expires_at: float | None = None) -> str: """Issue a target-verifiable bearer grant scoped to one room member.""" if len(secret) < 32: raise HostedRoomGrantError("room grant secret must be at least 32 bytes") - now = time.time() if issued_at is None else float(issued_at) + now = clock(issued_at) bounded_status_expiry = ( now + float(ttl_seconds if status_ttl_seconds is None else status_ttl_seconds) if status_expires_at is None - else float(status_expires_at) - ) + else float(status_expires_at)) if ( not math.isfinite(now) or ttl_seconds <= 0 or ttl_seconds > MAX_DISPATCH_GRANT_TTL_SECONDS or not math.isfinite(bounded_status_expiry) or bounded_status_expiry < now + float(ttl_seconds) - or bounded_status_expiry > now + MAX_STATUS_GRANT_TTL_SECONDS - ): + or bounded_status_expiry > now + MAX_STATUS_GRANT_TTL_SECONDS): raise HostedRoomGrantError("room grant lifetime is invalid") allowed = tuple(sorted(set(permissions))) if not allowed or not set(allowed) <= _GRANT_PERMISSIONS: @@ -501,19 +468,16 @@ def issue_room_grant( scope = { "grant_id": grant_id, "room_id": room_id, "home_install_id": home_install_id, "authority_gateway_id": authority_gateway_id, "authority_epoch": authority_epoch, - "member_id": member_id, "target_install_id": target_install_id, "target_profile": target_profile, - } + "member_id": member_id, "target_install_id": target_install_id, "target_profile": target_profile} payload = { "version": PROTOCOL_VERSION, **{name: _DISPATCH_FIELDS.get(name, _identifier)(scope[name], field=name) for name in _GRANT_SCOPE}, "execution_policy_digest": _digest( execution_policy_digest or execution_policy_mapping(target_profile=target_profile)["policy_digest"], - field="execution_policy_digest", - ), + field="execution_policy_digest"), "permissions": list(allowed), "issued_at": now, "expires_at": now + float(ttl_seconds), - "status_expires_at": bounded_status_expiry, - } + "status_expires_at": bounded_status_expiry} encoded = _canonical_json(payload) signature = hmac.new(secret, encoded, hashlib.sha256).digest() token = f"{_b64encode(encoded)}.{_b64encode(signature)}" @@ -524,8 +488,7 @@ def issue_room_grant( def verify_room_grant( secret: bytes, token: str, dispatch: HostedMemberDispatch, *, permission: str = "dispatch", - now: float | None = None, -) -> dict[str, Any]: + now: float | None = None) -> dict[str, Any]: """Verify one room grant against exact recipient dispatch coordinates.""" payload = decode_room_grant(secret, token, permission=permission, now=now) if payload["version"] != dispatch.protocol_version: @@ -536,9 +499,7 @@ def verify_room_grant( return payload -def decode_room_grant( - secret: bytes, token: str, *, permission: str, now: float | None = None -) -> dict[str, Any]: +def decode_room_grant(secret: bytes, token: str, *, permission: str, now: float | None = None) -> dict[str, Any]: """Verify grant signature, lifetime and operation without a dispatch.""" if not isinstance(token, str) or len(token.encode("utf-8")) > MAX_TOKEN_BYTES: raise HostedRoomGrantError("room grant is invalid") @@ -554,7 +515,7 @@ def decode_room_grant( raise HostedRoomGrantError("room grant payload is invalid") from exc if not isinstance(payload, dict) or frozenset(payload) not in {_GRANT_FIELDS, _GRANT_REFRESH_FIELDS}: raise HostedRoomGrantError("room grant fields are invalid") - checked_now = time.time() if now is None else float(now) + checked_now = clock(now) if not math.isfinite(checked_now): raise HostedRoomGrantError("room grant clock is invalid") try: @@ -575,18 +536,12 @@ def decode_room_grant( return payload -def room_grant_needs_dispatch_refresh( - token: str, *, now: float | None = None, leeway_seconds: float = 5 * 60 -) -> bool: - """Read only grant timing to schedule target-validated refresh. - - This deliberately does not establish trust; the target validates the - signature and immutable scope before issuing a replacement. - """ +def room_grant_needs_dispatch_refresh(token: str, *, now: float | None = None, leeway_seconds: float = 5 * 60) -> bool: + """Read only grant timing to schedule refresh; trust is established by the target, not here.""" try: payload = json.loads(_b64decode(_split_token(token)[0]).decode("ascii")) expires_at = float(payload["expires_at"]) - checked_now = time.time() if now is None else float(now) + checked_now = clock(now) return checked_now + max(0.0, float(leeway_seconds)) >= expires_at except Exception: return True diff --git a/gateway/hosted_room_policy_checkpoint.py b/gateway/hosted_room_policy_checkpoint.py index bcb8d738b2..da17a03a16 100644 --- a/gateway/hosted_room_policy_checkpoint.py +++ b/gateway/hosted_room_policy_checkpoint.py @@ -14,6 +14,7 @@ from pathlib import Path from typing import Any, Callable, Mapping from gateway import hosted_rooms +from gateway.hosted_rooms_common import compact_json MAX_ACTIVE_POLICY_EVENTS = 64 @@ -52,12 +53,8 @@ _SCHEMA_DDL = ( room_id TEXT PRIMARY KEY, schema_version INTEGER NOT NULL)""", ) -_ROOM_EVENT_COLUMNS = ( - "room_id, seq, event_id, kind, actor_json, authority_epoch, payload_json, created_at" -) -_DELETE_ACTIVE_EVENTS_SQL = ( - "DELETE FROM hosted_room_policy_events WHERE room_id=? AND discussion_event_id=?" -) +_ROOM_EVENT_COLUMNS = hosted_rooms._EVENT_COLUMNS +_DELETE_ACTIVE_EVENTS_SQL = ("DELETE FROM hosted_room_policy_events WHERE room_id=? AND discussion_event_id=?") _TRANSCRIPT_EVENTS_SQL = f"""WITH transcript_events(seq) AS ( SELECT seq FROM hosted_room_policy_transcript WHERE room_id=? AND thread_id=? UNION ALL @@ -79,13 +76,7 @@ class PolicySnapshot: watermarks: Mapping[tuple[str, str], int] -def _event_from_room_row(row: sqlite3.Row) -> dict[str, Any]: - return { - "room_id": str(row["room_id"]), "seq": int(row["seq"]), "event_id": str(row["event_id"]), - "kind": str(row["kind"]), "actor": json.loads(row["actor_json"]), - "authority_epoch": row["authority_epoch"], "payload": json.loads(row["payload_json"]), - "created_at": float(row["created_at"]), "idempotent": False, - } +_event_from_room_row = hosted_rooms._event_from_row def _require_room(conn: sqlite3.Connection, room_id: str) -> None: @@ -94,17 +85,14 @@ def _require_room(conn: sqlite3.Connection, room_id: str) -> None: def _settled_message( - conn: sqlite3.Connection, room_id: str, discussion_event_id: str, message_event_id: Any -) -> dict[str, Any] | None: + conn: sqlite3.Connection, room_id: str, discussion_event_id: str, message_event_id: Any) -> dict[str, Any] | None: """Return the indexed member message a ``turn.settled`` event committed, if it is in the projection.""" rows = conn.execute( "SELECT seq, event_json FROM hosted_room_policy_events WHERE room_id=? AND discussion_event_id=?", - (room_id, discussion_event_id), - ).fetchall() + (room_id, discussion_event_id)).fetchall() return next( (m for m in (json.loads(row["event_json"]) for row in rows) if m.get("event_id") == message_event_id), - None, - ) + None) class HostedRoomPolicyCheckpoint: @@ -127,43 +115,33 @@ class HostedRoomPolicyCheckpoint: @staticmethod def _store_active_event( - conn: sqlite3.Connection, *, event: Mapping[str, Any], thread_id: str, discussion_event_id: str - ) -> None: + conn: sqlite3.Connection, *, event: Mapping[str, Any], thread_id: str, discussion_event_id: str) -> None: conn.execute( """INSERT OR IGNORE INTO hosted_room_policy_events( room_id, thread_id, discussion_event_id, seq, event_json ) VALUES (?, ?, ?, ?, ?)""", - ( - event["room_id"], thread_id, discussion_event_id, int(event["seq"]), - json.dumps(dict(event), ensure_ascii=True, sort_keys=True, separators=(",", ":")), - ), - ) + (event["room_id"], thread_id, discussion_event_id, int(event["seq"]), compact_json(dict(event)))) @staticmethod def _store_transcript_event( - conn: sqlite3.Connection, *, event: Mapping[str, Any], thread_id: str, - settled_seq: int | None = None, - ) -> None: + conn: sqlite3.Connection, *, event: Mapping[str, Any], thread_id: str, settled_seq: int | None = None) -> None: conn.execute( """INSERT INTO hosted_room_policy_transcript( room_id, thread_id, seq, kind, settled_seq ) VALUES (?, ?, ?, ?, ?) ON CONFLICT(room_id, thread_id, seq) DO UPDATE SET settled_seq=COALESCE(excluded.settled_seq, hosted_room_policy_transcript.settled_seq)""", - (event["room_id"], thread_id, int(event["seq"]), str(event["kind"]), settled_seq), - ) + (event["room_id"], thread_id, int(event["seq"]), str(event["kind"]), settled_seq)) if event["kind"] in {"message.user", "message.member"}: cutoff = conn.execute( """SELECT seq FROM hosted_room_policy_transcript WHERE room_id=? AND thread_id=? AND kind IN ('message.user', 'message.member') ORDER BY seq DESC LIMIT 1 OFFSET ?""", - (event["room_id"], thread_id, MAX_THREAD_TRANSCRIPT_EVENTS - 1), - ).fetchone() + (event["room_id"], thread_id, MAX_THREAD_TRANSCRIPT_EVENTS - 1)).fetchone() if cutoff is not None: conn.execute( "DELETE FROM hosted_room_policy_transcript WHERE room_id=? AND thread_id=? AND seq None: """Migrate bounded committed thread history from the durable room log.""" @@ -172,9 +150,7 @@ class HostedRoomPolicyCheckpoint: settled_seq_by_message: dict[str, int] = {} for row in conn.execute( """SELECT seq, payload_json FROM hosted_room_events - WHERE room_id=? AND seq<=? AND kind='turn.settled' ORDER BY seq""", - (room_id, through_seq), - ): + WHERE room_id=? AND seq<=? AND kind='turn.settled' ORDER BY seq""", (room_id, through_seq)): message_event_id = str(json.loads(row["payload_json"]).get("message_event_id") or "") if message_event_id: settled_seq_by_message[message_event_id] = int(row["seq"]) @@ -182,8 +158,7 @@ class HostedRoomPolicyCheckpoint: f"""SELECT {_ROOM_EVENT_COLUMNS} FROM hosted_room_events WHERE room_id=? AND seq<=? AND kind IN ('message.user', 'message.member') ORDER BY seq""", - (room_id, through_seq), - ) + (room_id, through_seq)) for row in rows: if row["kind"] == "message.member" and row["event_id"] not in settled_seq_by_message: continue @@ -192,38 +167,30 @@ class HostedRoomPolicyCheckpoint: if thread_id: self._store_transcript_event( conn, event=event, thread_id=thread_id, - settled_seq=settled_seq_by_message.get(str(row["event_id"])), - ) + settled_seq=settled_seq_by_message.get(str(row["event_id"]))) def _discussion_events( self, conn: sqlite3.Connection, *, room_id: str, thread_id: str, discussion_event_id: str, - bound_error: str, - ) -> list[dict[str, Any]]: + bound_error: str) -> list[dict[str, Any]]: """Merge the thread transcript with the active projection, ordered by seq.""" active_rows = conn.execute( """SELECT event_json FROM hosted_room_policy_events WHERE room_id=? AND discussion_event_id=? ORDER BY seq LIMIT ?""", - (room_id, discussion_event_id, MAX_ACTIVE_POLICY_EVENTS + 1), - ).fetchall() + (room_id, discussion_event_id, MAX_ACTIVE_POLICY_EVENTS + 1)).fetchall() if len(active_rows) > MAX_ACTIVE_POLICY_EVENTS: raise RuntimeError(bound_error) - rows = conn.execute( - _TRANSCRIPT_EVENTS_SQL, (room_id, thread_id, room_id, thread_id, room_id) - ).fetchall() + rows = conn.execute(_TRANSCRIPT_EVENTS_SQL, (room_id, thread_id, room_id, thread_id, room_id)).fetchall() events_by_seq = { int(event["seq"]): event for event in ( *(_event_from_room_row(row) for row in rows), - *(json.loads(row["event_json"]) for row in active_rows), - ) - } + *(json.loads(row["event_json"]) for row in active_rows))} return [events_by_seq[seq] for seq in sorted(events_by_seq)] # -- per-kind projection handlers (dispatched by _apply_event) ----------- def _apply_user_message( - self, conn: sqlite3.Connection, event: Mapping[str, Any], payload: Mapping[str, Any] - ) -> None: + self, conn: sqlite3.Connection, event: Mapping[str, Any], payload: Mapping[str, Any]) -> None: room_id = str(event["room_id"]) thread_id = str(payload.get("thread_id") or "") event_id = str(event.get("event_id") or "") @@ -236,14 +203,12 @@ class HostedRoomPolicyCheckpoint: ON CONFLICT(room_id, thread_id) DO UPDATE SET discussion_event_id=excluded.discussion_event_id, latest_user_seq=excluded.latest_user_seq, completed=0""", - (room_id, thread_id, event_id, int(event["seq"])), - ) + (room_id, thread_id, event_id, int(event["seq"]))) self._store_active_event(conn, event=event, thread_id=thread_id, discussion_event_id=event_id) self._store_transcript_event(conn, event=event, thread_id=thread_id) def _apply_discussion_event( - self, conn: sqlite3.Connection, event: Mapping[str, Any], payload: Mapping[str, Any] - ) -> None: + self, conn: sqlite3.Connection, event: Mapping[str, Any], payload: Mapping[str, Any]) -> None: """Index member messages and terminal turn outcomes of a known discussion.""" room_id = str(event["room_id"]) seq = int(event["seq"]) @@ -252,26 +217,20 @@ class HostedRoomPolicyCheckpoint: discussion_event_id = str(payload.get("discussion_event_id") or "") source = conn.execute( "SELECT 1 FROM hosted_room_policy_events WHERE room_id=? AND discussion_event_id=? LIMIT 1", - (room_id, discussion_event_id), - ).fetchone() + (room_id, discussion_event_id)).fetchone() if source is None: return - self._store_active_event( - conn, event=event, thread_id=thread_id, discussion_event_id=discussion_event_id - ) + self._store_active_event(conn, event=event, thread_id=thread_id, discussion_event_id=discussion_event_id) if kind not in _TERMINAL_KINDS: return task_id = str(payload.get("task_id") or "") - execution_generation = ( - int(payload.get("execution_generation") or 0) if kind == "turn.deferred" else 0 - ) + execution_generation = (int(payload.get("execution_generation") or 0) if kind == "turn.deferred" else 0) if task_id: conn.execute( """INSERT OR IGNORE INTO hosted_room_policy_publications( room_id, task_id, kind, execution_generation, seq ) VALUES (?, ?, ?, ?, ?)""", - (room_id, task_id, kind, execution_generation, seq), - ) + (room_id, task_id, kind, execution_generation, seq)) member_id = str(payload.get("member_id") or "") seen_through_seq = int(payload.get("seen_through_seq") or 0) if kind == "turn.settled" and payload.get("message_event_id"): @@ -286,35 +245,26 @@ class HostedRoomPolicyCheckpoint: ) VALUES (?, ?, ?, ?) ON CONFLICT(room_id, thread_id, member_id) DO UPDATE SET seen_through_seq=MAX(hosted_room_policy_watermarks.seen_through_seq, excluded.seen_through_seq)""", - (room_id, thread_id, member_id, seen_through_seq), - ) + (room_id, thread_id, member_id, seen_through_seq)) def _apply_room_activity( - self, conn: sqlite3.Connection, event: Mapping[str, Any], payload: Mapping[str, Any] - ) -> None: + self, conn: sqlite3.Connection, event: Mapping[str, Any], payload: Mapping[str, Any]) -> None: room_id = str(event["room_id"]) thread_id = str(payload.get("thread_id") or "") discussion_event_id = str(payload.get("discussion_event_id") or "") conn.execute(_DELETE_ACTIVE_EVENTS_SQL, (room_id, discussion_event_id)) - conn.execute( - "DELETE FROM hosted_room_policy_threads WHERE room_id=? AND thread_id=?", - (room_id, thread_id), - ) + conn.execute("DELETE FROM hosted_room_policy_threads WHERE room_id=? AND thread_id=?", (room_id, thread_id)) def _apply_stop_requested( - self, conn: sqlite3.Connection, event: Mapping[str, Any], payload: Mapping[str, Any] - ) -> None: + self, conn: sqlite3.Connection, event: Mapping[str, Any], payload: Mapping[str, Any]) -> None: conn.execute( """UPDATE hosted_room_policy_cursors SET stopped_through_seq=MAX(stopped_through_seq, ?) WHERE room_id=?""", - (int(event["seq"]), str(event["room_id"])), - ) + (int(event["seq"]), str(event["room_id"]))) _APPLY_BY_KIND: dict[str, Callable[..., None]] = { - "message.user": _apply_user_message, - "message.member": _apply_discussion_event, - **dict.fromkeys(_TERMINAL_KINDS, _apply_discussion_event), - "room.activity": _apply_room_activity, + "message.user": _apply_user_message, "message.member": _apply_discussion_event, + **dict.fromkeys(_TERMINAL_KINDS, _apply_discussion_event), "room.activity": _apply_room_activity, "room.stop_requested": _apply_stop_requested, } @@ -331,25 +281,18 @@ class HostedRoomPolicyCheckpoint: conn.execute( """INSERT OR IGNORE INTO hosted_room_policy_cursors( room_id, through_seq, stopped_through_seq, updated_at - ) VALUES (?, 0, 0, 0)""", - (room_id,), - ) - row = conn.execute( - "SELECT through_seq FROM hosted_room_policy_cursors WHERE room_id=?", (room_id,) - ).fetchone() + ) VALUES (?, 0, 0, 0)""", (room_id,)) + row = conn.execute("SELECT through_seq FROM hosted_room_policy_cursors WHERE room_id=?", (room_id,)).fetchone() cursor = int(row["through_seq"]) transcript_state = conn.execute( - "SELECT schema_version FROM hosted_room_policy_transcript_state WHERE room_id=?", - (room_id,), - ).fetchone() + "SELECT schema_version FROM hosted_room_policy_transcript_state WHERE room_id=?", (room_id,)).fetchone() if transcript_state is None or int(transcript_state["schema_version"]) < _TRANSCRIPT_SCHEMA_VERSION: self._backfill_transcript(conn, room_id=room_id, through_seq=cursor) conn.execute( """INSERT INTO hosted_room_policy_transcript_state(room_id, schema_version) VALUES (?, ?) ON CONFLICT(room_id) DO UPDATE SET schema_version=excluded.schema_version""", - (room_id, _TRANSCRIPT_SCHEMA_VERSION), - ) + (room_id, _TRANSCRIPT_SCHEMA_VERSION)) return cursor def sync(self, *, room_id: str, latest_seq: int) -> int: @@ -362,8 +305,7 @@ class HostedRoomPolicyCheckpoint: while cursor < latest_seq: page = hosted_rooms.read_events( - self.db_path, room_id=room_id, since_seq=cursor, limit=hosted_rooms.MAX_LOG_LIMIT - ) + self.db_path, room_id=room_id, since_seq=cursor, limit=hosted_rooms.MAX_LOG_LIMIT) rows = [event for event in page.get("events", []) if isinstance(event, Mapping)] next_cursor = int(page.get("cursor") or cursor) if not rows or next_cursor <= cursor: @@ -375,8 +317,7 @@ class HostedRoomPolicyCheckpoint: self._apply_event(conn, event) updated = conn.execute( "UPDATE hosted_room_policy_cursors SET through_seq=?, updated_at=? WHERE room_id=?", - (next_cursor, float(rows[-1].get("created_at") or 0), room_id), - ) + (next_cursor, float(rows[-1].get("created_at") or 0), room_id)) if updated.rowcount != 1: raise RuntimeError("room policy cursor disappeared during replay") cursor = next_cursor @@ -387,43 +328,29 @@ class HostedRoomPolicyCheckpoint: through_seq = self.sync(room_id=room_id, latest_seq=latest_seq) with self._connect() as conn: cursor = conn.execute( - "SELECT stopped_through_seq FROM hosted_room_policy_cursors WHERE room_id=?", - (room_id,), - ).fetchone() + "SELECT stopped_through_seq FROM hosted_room_policy_cursors WHERE room_id=?", (room_id,)).fetchone() stopped_through_seq = int(cursor["stopped_through_seq"]) thread = conn.execute( """SELECT thread_id, discussion_event_id FROM hosted_room_policy_threads WHERE room_id=? AND completed=0 AND latest_user_seq>? - ORDER BY latest_user_seq, thread_id LIMIT 1""", - (room_id, stopped_through_seq), - ).fetchone() + ORDER BY latest_user_seq, thread_id LIMIT 1""", (room_id, stopped_through_seq)).fetchone() if thread is None: return PolicySnapshot( through_seq=through_seq, stopped_through_seq=stopped_through_seq, - events=(), watermarks={}, - ) + events=(), watermarks={}) thread_id = str(thread["thread_id"]) events = self._discussion_events( conn, room_id=room_id, thread_id=thread_id, discussion_event_id=str(thread["discussion_event_id"]), - bound_error="active room policy projection exceeded its bound", - ) + bound_error="active room policy projection exceeded its bound") watermark_rows = conn.execute( """SELECT member_id, seen_through_seq FROM hosted_room_policy_watermarks - WHERE room_id=? AND thread_id=?""", - (room_id, thread_id), - ).fetchall() + WHERE room_id=? AND thread_id=?""", (room_id, thread_id)).fetchall() return PolicySnapshot( through_seq=through_seq, stopped_through_seq=stopped_through_seq, events=tuple(events), - watermarks={ - (thread_id, str(row["member_id"])): int(row["seen_through_seq"]) - for row in watermark_rows - }, - ) + watermarks={(thread_id, str(row["member_id"])): int(row["seen_through_seq"]) for row in watermark_rows}) - def publication_exists( - self, *, room_id: str, task_id: str, status: str, execution_generation: int - ) -> bool: + def publication_exists(self, *, room_id: str, task_id: str, status: str, execution_generation: int) -> bool: """Return whether one exact driver outcome is already in the room log.""" if status == "deferred": sql = """SELECT 1 FROM hosted_room_policy_publications @@ -441,25 +368,20 @@ class HostedRoomPolicyCheckpoint: with self._connect() as conn: source = conn.execute( "SELECT discussion_event_id, thread_id FROM hosted_room_policy_events WHERE room_id=? AND seq=?", - (room_id, source_event_seq), - ).fetchone() + (room_id, source_event_seq)).fetchone() if source is None: return [] return self._discussion_events( conn, room_id=room_id, thread_id=str(source["thread_id"]), discussion_event_id=str(source["discussion_event_id"]), - bound_error="task policy projection exceeded its bound", - ) + bound_error="task policy projection exceeded its bound") def compact_completed(self, *, room_id: str) -> None: """Drop any completed projections left by an interrupted sync.""" with self._connect() as conn: completed = conn.execute( "SELECT discussion_event_id FROM hosted_room_policy_threads WHERE room_id=? AND completed=1", - (room_id,), - ).fetchall() + (room_id,)).fetchall() for row in completed: conn.execute(_DELETE_ACTIVE_EVENTS_SQL, (room_id, str(row["discussion_event_id"]))) - conn.execute( - "DELETE FROM hosted_room_policy_threads WHERE room_id=? AND completed=1", (room_id,) - ) + conn.execute("DELETE FROM hosted_room_policy_threads WHERE room_id=? AND completed=1", (room_id,)) diff --git a/gateway/hosted_room_replicas.py b/gateway/hosted_room_replicas.py index 41815c8e5a..2ab23d22a8 100644 --- a/gateway/hosted_room_replicas.py +++ b/gateway/hosted_room_replicas.py @@ -1,41 +1,35 @@ """Replica store and takeover primitives for hosted Group Chat rooms. -The authority gateway owns a room's ordered log in ``gateway/hosted_rooms.py``; -this module gives every OTHER participant gateway a durable local copy plus the -fenced primitives to continue the room when the authority host dies: -``ingest_page()`` (idempotent, gap- and epoch-regression-safe replay), -``promote_replica()`` (resume locally at ``epoch + 1`` with a lineage-proving -``authority.claimed`` event) and ``demote_room()`` (a returning stale authority -records ``authority.lost`` when shown proof of a newer epoch). - -Storage primitives only: none decide *when* takeover is safe. The caller must -establish that the previous owner can no longer commit before promoting. +Non-authority gateways keep a durable local copy of the room log (``ingest_page()``: idempotent, +gap- and epoch-regression-safe) plus fenced primitives to continue the room when the authority host +dies: ``promote_replica()`` resumes locally at ``epoch + 1`` with a lineage-proving ``authority.claimed`` +event; ``demote_room()`` records ``authority.lost`` when a returning stale authority is shown a newer +epoch. Storage primitives only: the caller decides *when* takeover is safe. """ from __future__ import annotations import json import sqlite3 -import time from contextlib import contextmanager +from functools import partial from pathlib import Path from typing import Any, Iterator from gateway.hosted_rooms import ( MAX_ACTOR_ID_CHARS, - MAX_EVENT_JSON_BYTES, - MAX_ROOM_ID_CHARS, HostedRoomError, RoomConflictError, - _canonical_json, + _actor_json, _connect, + _payload_json, + _room_id, _transaction, _validate_identifier, _validate_members, _validate_room_name, - local_authority_gateway_id, -) -from gateway.hosted_rooms_common import positive_int, utf8_len + local_authority_gateway_id) +from gateway.hosted_rooms_common import bounded_int, clock, utf8_len MAX_REPLICA_ROOMS = 256 MAX_REPLICA_EVENT_BYTES = 256 * 1024 * 1024 @@ -46,16 +40,13 @@ _INSERT_REPLICA_EVENT = f"INSERT INTO hosted_room_replica_events {_EVENT_COLUMNS _SELECT_REPLICA = "SELECT * FROM hosted_room_replicas WHERE room_id=?" -class ReplicaError(HostedRoomError): - """Base class for invalid or conflicting replica operations.""" +class ReplicaError(HostedRoomError): """Base class for invalid or conflicting replica operations.""" -class ReplicaGapError(ReplicaError): - """A page does not start at the replica's next expected sequence.""" +class ReplicaGapError(ReplicaError): """A page does not start at the replica's next expected sequence.""" -class ReplicaEpochRegressionError(ReplicaError): - """A page or demotion carries an older authority epoch than stored.""" +class ReplicaEpochRegressionError(ReplicaError): """A page or demotion carries an older authority epoch than stored.""" def _initialize_replica_schema(conn: sqlite3.Connection) -> None: @@ -83,8 +74,7 @@ def _initialize_replica_schema(conn: sqlite3.Connection) -> None: def _replica_transaction(db_path: Path | str) -> Iterator[sqlite3.Connection]: """Ensure the replica schema (own autocommit connection), then open an IMMEDIATE transaction. - The schema DDL is re-run inside the transaction too; that double init is the - established statement order and is intentionally preserved. + The DDL is deliberately re-run inside the transaction: that double init is the established statement order. """ conn = _connect(db_path) try: @@ -97,28 +87,19 @@ def _replica_transaction(db_path: Path | str) -> Iterator[sqlite3.Connection]: yield conn -def _room_id(value: Any) -> str: - return _validate_identifier(value, label="room_id", max_chars=MAX_ROOM_ID_CHARS) - - -def _positive_int(value: Any, message: str) -> int: - return positive_int(value, error=ReplicaError, message=message) +_positive_int = partial(bounded_int, error=ReplicaError, low=1) def _control_event_json(payload: dict[str, Any]) -> tuple[str, str]: """Canonical (actor_json, payload_json) for a system authority-control event.""" - return ( - _canonical_json(_SYSTEM_ACTOR, label="actor", max_bytes=4 * 1024), - _canonical_json(payload, label="payload", max_bytes=MAX_EVENT_JSON_BYTES), - ) + return _actor_json(_SYSTEM_ACTOR), _payload_json(payload) def _event_bytes(event: dict[str, Any]) -> int: return utf8_len( str(event["event_id"]), str(event["kind"]), json.dumps(event["actor"], ensure_ascii=False, separators=(",", ":")), - json.dumps(event["payload"], ensure_ascii=False, separators=(",", ":")), - ) + json.dumps(event["payload"], ensure_ascii=False, separators=(",", ":"))) def _validate_page(page: Any) -> tuple[list[dict[str, Any]], dict[str, Any]]: @@ -131,14 +112,13 @@ def _validate_page(page: Any) -> tuple[list[dict[str, Any]], dict[str, Any]]: if not isinstance(authority, dict): raise ReplicaError("page.authority is required for replication") gateway_id = _validate_identifier( - authority.get("gateway_id"), label="page.authority.gateway_id", max_chars=MAX_ACTOR_ID_CHARS - ) - epoch = _positive_int(authority.get("epoch"), "page.authority.epoch must be a positive integer") + authority.get("gateway_id"), label="page.authority.gateway_id", max_chars=MAX_ACTOR_ID_CHARS) + epoch = _positive_int(authority.get("epoch"), message="page.authority.epoch must be a positive integer") previous_seq: int | None = None for event in events: if not isinstance(event, dict): raise ReplicaError("page events must be objects") - seq = _positive_int(event.get("seq"), "event.seq must be a positive integer") + seq = _positive_int(event.get("seq"), message="event.seq must be a positive integer") if previous_seq is not None and seq != previous_seq + 1: raise ReplicaGapError("page events must be contiguous") previous_seq = seq @@ -156,9 +136,7 @@ def _replica_row_state(conn: sqlite3.Connection, room_id: str) -> tuple[sqlite3. """Return (row, stored_epoch, last_seq, stored_bytes); a new room is admitted only under the room cap.""" row = conn.execute( """SELECT authority_gateway_id, authority_epoch, last_seq, latest_seq, event_bytes - FROM hosted_room_replicas WHERE room_id=?""", - (room_id,), - ).fetchone() + FROM hosted_room_replicas WHERE room_id=?""", (room_id,)).fetchone() if row is None: count = conn.execute("SELECT COUNT(*) FROM hosted_room_replicas").fetchone()[0] if int(count) >= MAX_REPLICA_ROOMS: @@ -169,8 +147,7 @@ def _replica_row_state(conn: sqlite3.Connection, room_id: str) -> tuple[sqlite3. def _store_replica( conn: sqlite3.Connection, *, is_new: bool, room_id: str, room_name: str, members_json: str, - authority: dict[str, Any], new_last: int, latest_seq: int, added_bytes: int, now: float, -) -> None: + authority: dict[str, Any], new_last: int, latest_seq: int, added_bytes: int, now: float) -> None: """INSERT the replica row for a new room, else UPDATE it (event_bytes accumulates).""" values = (room_name, members_json, authority["gateway_id"], authority["epoch"], new_last, max(latest_seq, new_last)) if is_new: @@ -178,30 +155,23 @@ def _store_replica( """INSERT INTO hosted_room_replicas (room_id, name, members_json, authority_gateway_id, authority_epoch, last_seq, latest_seq, event_bytes, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", - (room_id, *values, added_bytes, now, now), - ) + (room_id, *values, added_bytes, now, now)) else: conn.execute( """UPDATE hosted_room_replicas SET name=?, members_json=?, authority_gateway_id=?, authority_epoch=?, last_seq=?, latest_seq=?, event_bytes=event_bytes+?, - updated_at=? WHERE room_id=?""", - (*values, added_bytes, now, room_id), - ) + updated_at=? WHERE room_id=?""", (*values, added_bytes, now, room_id)) def ingest_page( db_path: Path | str, *, room_id: Any, room_name: Any, members: Any, page: Any, now: float | None = None ) -> dict[str, Any]: - """Persist one replay page idempotently; refuses seq gaps and epoch regressions. - - ``page`` is the verbatim ``groups.log`` (``read_events()``) result whose - ``authority`` stamp proves lineage. - """ + """Persist one verbatim ``read_events()`` page idempotently; refuses seq gaps and epoch regressions.""" room_id = _room_id(room_id) room_name = _validate_room_name(room_name) _, members_json = _validate_members(members) events, authority = _validate_page(page) - now = time.time() if now is None else float(now) + now = clock(now) with _replica_transaction(db_path) as conn: row, stored_epoch, last_seq, stored_bytes = _replica_row_state(conn, room_id) @@ -216,15 +186,13 @@ def ingest_page( size = _event_bytes(event) if stored_bytes + added_bytes + size > MAX_REPLICA_EVENT_BYTES: raise ReplicaError("replica event storage exhausted") - actor_json = _canonical_json(event["actor"], label="actor", max_bytes=4 * 1024) - payload_json = _canonical_json(event["payload"], label="payload", max_bytes=MAX_EVENT_JSON_BYTES) + actor_json = _actor_json(event["actor"]) + payload_json = _payload_json(event["payload"]) conn.execute( _INSERT_REPLICA_EVENT, ( room_id, int(event["seq"]), event["event_id"], event["kind"], actor_json, - event.get("authority_epoch"), payload_json, float(event.get("created_at") or now), - ), - ) + event.get("authority_epoch"), payload_json, float(event.get("created_at") or now))) added_bytes += size new_last = int(new_events[-1]["seq"]) if new_events else last_seq latest_seq = page.get("latest_seq") @@ -232,13 +200,9 @@ def ingest_page( latest_seq = new_last _store_replica( conn, is_new=row is None, room_id=room_id, room_name=room_name, members_json=members_json, - authority=authority, new_last=new_last, latest_seq=latest_seq, added_bytes=added_bytes, now=now, - ) + authority=authority, new_last=new_last, latest_seq=latest_seq, added_bytes=added_bytes, now=now) return { - "room_id": room_id, - "stored_seq": new_last, - "ingested": len(new_events), - "authority": authority, + "room_id": room_id, "stored_seq": new_last, "ingested": len(new_events), "authority": authority, "caught_up": new_last >= max(latest_seq, new_last), } @@ -251,15 +215,10 @@ def replica_state(db_path: Path | str, *, room_id: Any) -> dict[str, Any]: if row is None: raise ReplicaError("replica not found") return { - "room_id": row["room_id"], - "name": row["name"], - "members": json.loads(row["members_json"]), + "room_id": row["room_id"], "name": row["name"], "members": json.loads(row["members_json"]), "authority": {"gateway_id": row["authority_gateway_id"], "epoch": int(row["authority_epoch"])}, - "last_seq": int(row["last_seq"]), - "latest_seq": int(row["latest_seq"]), - "event_bytes": int(row["event_bytes"]), - "created_at": float(row["created_at"]), - "updated_at": float(row["updated_at"]), + "last_seq": int(row["last_seq"]), "latest_seq": int(row["latest_seq"]), "event_bytes": int(row["event_bytes"]), + "created_at": float(row["created_at"]), "updated_at": float(row["updated_at"]), } @@ -268,15 +227,14 @@ def promote_replica( ) -> dict[str, Any]: """Continue a replicated room on THIS gateway at ``epoch + 1``. - Copies the replica log into the authoritative store and appends a - lineage-proving ``authority.claimed`` event; wherever the claim replicates, - the old epoch is stale and every fenced primitive rejects it. The caller - decides takeover is safe; this only makes it atomic and provable. + Copies the replica log into the authoritative store and appends a lineage-proving + ``authority.claimed`` event, so wherever the claim replicates the old epoch is stale and every + fenced primitive rejects it. The caller decides takeover is safe; this makes it atomic and provable. """ room_id = _room_id(room_id) if not isinstance(reason, str) or not reason or len(reason) > 200: raise ReplicaError("reason must be a non-empty string of at most 200 chars") - now = time.time() if now is None else float(now) + now = clock(now) local_gateway = local_authority_gateway_id() with _replica_transaction(db_path) as conn: @@ -296,11 +254,8 @@ def promote_replica( claim_seq = int(replica["last_seq"]) + 1 claim_event_id = f"system:authority-claimed:{target_epoch}" claim_actor_json, claim_payload_json = _control_event_json({ - "previous_gateway_id": previous_gateway, - "authority_gateway_id": local_gateway, - "authority_epoch": target_epoch, - "promoted_from_replica": True, - "reason": reason, + "previous_gateway_id": previous_gateway, "authority_gateway_id": local_gateway, + "authority_epoch": target_epoch, "promoted_from_replica": True, "reason": reason, }) claim_bytes = utf8_len(claim_event_id, "authority.claimed", claim_actor_json, claim_payload_json) @@ -311,31 +266,22 @@ def promote_replica( VALUES (?, ?, ?, ?, ?, ?, ?, 1, ?, ?, NULL)""", ( room_id, replica["name"], replica["members_json"], local_gateway, target_epoch, - claim_seq + 1, int(replica["event_bytes"]) + claim_bytes, now, now, - ), - ) + claim_seq + 1, int(replica["event_bytes"]) + claim_bytes, now, now)) conn.execute( f"""INSERT INTO hosted_room_events {_EVENT_COLUMNS} SELECT room_id, seq, event_id, kind, actor_json, authority_epoch, payload_json, created_at FROM hosted_room_replica_events WHERE room_id=?""", - (room_id,), - ) + (room_id,)) conn.execute( _INSERT_ROOM_EVENT, ( room_id, claim_seq, claim_event_id, "authority.claimed", - claim_actor_json, target_epoch, claim_payload_json, now, - ), - ) + claim_actor_json, target_epoch, claim_payload_json, now)) conn.execute("DELETE FROM hosted_room_replica_events WHERE room_id=?", (room_id,)) conn.execute("DELETE FROM hosted_room_replicas WHERE room_id=?", (room_id,)) return { - "room_id": room_id, - "authority_gateway_id": local_gateway, - "authority_epoch": target_epoch, - "previous_gateway_id": previous_gateway, - "previous_epoch": previous_epoch, - "claim_seq": claim_seq, + "room_id": room_id, "authority_gateway_id": local_gateway, "authority_epoch": target_epoch, + "previous_gateway_id": previous_gateway, "previous_epoch": previous_epoch, "claim_seq": claim_seq, "latest_seq": claim_seq, } @@ -345,34 +291,28 @@ def demote_room( ) -> dict[str, Any]: """Fence THIS gateway's stale room authority against a proven newer epoch. - Called when a returning gateway observes (replicated ``authority.claimed`` - or a transport rejection) that another gateway owns the room at a higher - epoch. Appends ``authority.lost`` and adopts the observed lineage so no - local send can commit at the stale epoch. Idempotent per lineage. + When a returning gateway observes (replicated ``authority.claimed`` or a transport rejection) that + another gateway owns the room at a higher epoch, append ``authority.lost`` and adopt the observed + lineage so no local send can commit at the stale epoch. Idempotent per lineage. """ room_id = _room_id(room_id) observed_gateway_id = _validate_identifier( - observed_gateway_id, label="observed_gateway_id", max_chars=MAX_ACTOR_ID_CHARS - ) - observed_epoch = _positive_int(observed_epoch, "observed_epoch must be a positive integer") - now = time.time() if now is None else float(now) + observed_gateway_id, label="observed_gateway_id", max_chars=MAX_ACTOR_ID_CHARS) + observed_epoch = _positive_int(observed_epoch, message="observed_epoch must be a positive integer") + now = clock(now) local_gateway = local_authority_gateway_id() with _transaction(db_path, immediate=True) as conn: row = conn.execute( """SELECT authority_gateway_id, authority_epoch, next_seq - FROM hosted_rooms WHERE room_id=? AND disbanded_at IS NULL""", - (room_id,), - ).fetchone() + FROM hosted_rooms WHERE room_id=? AND disbanded_at IS NULL""", (room_id,)).fetchone() if row is None: raise ReplicaError("room not found in the local authoritative store") current_gateway = str(row["authority_gateway_id"]) current_epoch = int(row["authority_epoch"]) if current_gateway == observed_gateway_id and current_epoch == observed_epoch: return { - "room_id": room_id, - "authority_gateway_id": current_gateway, - "authority_epoch": current_epoch, + "room_id": room_id, "authority_gateway_id": current_gateway, "authority_epoch": current_epoch, "idempotent": True, } if observed_epoch <= current_epoch: @@ -382,24 +322,18 @@ def demote_room( lost_actor_json, lost_payload_json = _control_event_json({ "previous_gateway_id": current_gateway, "authority_gateway_id": observed_gateway_id, - "authority_epoch": observed_epoch, - }) + "authority_epoch": observed_epoch}) conn.execute( _INSERT_ROOM_EVENT, ( room_id, int(row["next_seq"]), f"system:authority-lost:{observed_epoch}", - "authority.lost", lost_actor_json, observed_epoch, lost_payload_json, now, - ), - ) + "authority.lost", lost_actor_json, observed_epoch, lost_payload_json, now)) conn.execute( """UPDATE hosted_rooms SET authority_gateway_id=?, authority_epoch=?, next_seq=next_seq+1, revision=revision+1, updated_at=? WHERE room_id=?""", - (observed_gateway_id, observed_epoch, now, room_id), - ) + (observed_gateway_id, observed_epoch, now, room_id)) return { - "room_id": room_id, - "authority_gateway_id": observed_gateway_id, - "authority_epoch": observed_epoch, + "room_id": room_id, "authority_gateway_id": observed_gateway_id, "authority_epoch": observed_epoch, "idempotent": False, } diff --git a/gateway/hosted_rooms.py b/gateway/hosted_rooms.py index 2daf75d97b..7ae15b9039 100644 --- a/gateway/hosted_rooms.py +++ b/gateway/hosted_rooms.py @@ -1,12 +1,9 @@ """Durable state for gateway-hosted Bot Mode rooms. -This module owns only hosted-room identity and its append-only event log. It -does not deliver events, lease relay work, or run agent turns; those belong to -the relay and the hosted-room driver, so the room log composes with a durable -relay without creating a second transport queue. - -The caller supplies the database path so tests and alternate gateway layouts -can isolate state. Production handlers use the gateway's root ``state.db``. +Owns only hosted-room identity and its append-only event log; delivery, relay leasing +and agent turns belong to the relay and the hosted-room driver, so the log composes with +a durable relay without a second transport queue. Callers supply the database path +(production handlers use the gateway's root ``state.db``). """ from __future__ import annotations @@ -15,20 +12,13 @@ import hashlib import json import re import sqlite3 -import time +from functools import partial from pathlib import Path from typing import Any, Mapping, NoReturn from gateway.hosted_rooms_common import ( - canonical_json, - identifier, - non_negative_int, - open_sqlite, - positive_int, - table_columns, - table_exists, - transaction, -) + bounded_int, canonical_json, clock as _now, compact_json, connect, identifier, open_sqlite, + table_columns, table_exists, transaction) PROTOCOL_VERSION = 2 MAX_ROOM_ID_CHARS = 128 @@ -57,29 +47,25 @@ CONTROL_EVENT_BYTE_RESERVE = 1024 * 1024 _JOURNAL_MODE_LOCK_RETRIES = 8 _EVENT_KIND_RE = re.compile(r"^[a-z][a-z0-9_.-]*$") -_CONTROL_EVENT_KINDS = frozenset({ - "authority.claimed", "authority.lost", "room.disbanded", "room.stop_requested", -}) +_CONTROL_EVENT_KINDS = frozenset({"authority.claimed", "authority.lost", "room.disbanded", "room.stop_requested"}) _EVENT_KINDS_BY_ACTOR = { "user": frozenset({"message.user"}), "member": frozenset({"message.member"}), "gateway": frozenset({ "member.unavailable", "room.activity", "room.stop_requested", "turn.deferred", - "turn.reassigned", "turn.cancelled", "turn.failed", "turn.settled", "turn.started", - }), + "turn.reassigned", "turn.cancelled", "turn.failed", "turn.settled", "turn.started"}), "system": frozenset({ "authority.claimed", "authority.lost", "room.created", "room.disbanded", - "room.members_changed", "room.renamed", - }), -} -_ACTOR_FIELDS = frozenset({"kind", "id", "display_name", "profile", "connection_id"}) + "room.members_changed", "room.renamed"})} +_OPTIONAL_ACTOR_FIELDS = ( + ("display_name", MAX_ACTOR_LABEL_CHARS), ("profile", MAX_ACTOR_ID_CHARS), ("connection_id", MAX_ACTOR_ID_CHARS)) +_ACTOR_FIELDS = frozenset({"kind", "id", *(field for field, _ in _OPTIONAL_ACTOR_FIELDS)}) # --- schema ----------------------------------------------------------------- _REMOTE_RUN_IDENTITY_COLUMNS = ( "room_id", "home_install_id", "authority_gateway_id", "authority_epoch", "member_id", - "target_install_id", "target_profile", "task_id", "execution_generation", -) + "target_install_id", "target_profile", "task_id", "execution_generation") _REMOTE_RUNS_BODY = """ room_id TEXT NOT NULL, home_install_id TEXT NOT NULL, @@ -165,74 +151,47 @@ _SCHEMA_DDL = ( PRIMARY KEY (room_id, member_id, target_profile) )""", ) -# (table, required columns) in the order _schema_is_current probes them. -_REQUIRED_COLUMNS = ( - ("hosted_rooms", frozenset({ - "room_id", "name", "members_json", "authority_gateway_id", "authority_epoch", - "next_seq", "event_bytes", "revision", "created_at", "updated_at", "disbanded_at", - })), - ("hosted_room_events", frozenset({ - "room_id", "seq", "event_id", "kind", "actor_json", "authority_epoch", - "payload_json", "created_at", - })), - ("hosted_room_retired_ids", frozenset({"room_id", "retired_at"})), - ("hosted_room_links", frozenset({ - "room_id", "member_id", "target_url", "target_profile", "grant", "catalog_json", - "cancellation_scope_id", "trace_id", "transport_security", "status", "updated_at", - })), - ("hosted_room_remote_runs", frozenset( - (*_REMOTE_RUN_IDENTITY_COLUMNS, "run_id", "session_id", "created_at", "updated_at") - )), - ("hosted_room_revoked_grants", frozenset({"scope_key", "expires_at", "revoked_before"})), - ("hosted_room_peer_reservations", frozenset({ - "room_id", "member_id", "target_profile", "authority_gateway_id", "authority_epoch", - "expires_at", "revoked_at", "created_at", "updated_at", - })), -) +# (table, required columns) parsed from the DDL, in the order _schema_is_current probes them. +_REQUIRED_COLUMNS = tuple( + ( + re.search(r"EXISTS (\w+)", ddl).group(1), + frozenset(re.findall(r"^\s*(\w+) (?:TEXT|INTEGER|REAL)\b", ddl.split("(", 1)[1], re.M))) + for ddl in _SCHEMA_DDL) _REMOTE_RUN_SCHEMA_COLUMNS = _REQUIRED_COLUMNS[4][1] # --- SQL fragments (statement text must stay byte-stable after whitespace normalisation) --- -_EVENT_COLUMNS = ( - "room_id, seq, event_id, kind, actor_json, authority_epoch, payload_json, created_at" -) +_EVENT_COLUMNS = ("room_id, seq, event_id, kind, actor_json, authority_epoch, payload_json, created_at") _SELECT_EVENT = f"SELECT {_EVENT_COLUMNS} FROM hosted_room_events WHERE room_id=? AND event_id=?" _INSERT_EVENT = ( - f"INSERT INTO hosted_room_events ({_EVENT_COLUMNS}) VALUES (?, ?, ?, ?, ?, ?, ?, ?)" -) + f"INSERT INTO hosted_room_events ({_EVENT_COLUMNS}) VALUES (?, ?, ?, ?, ?, ?, ?, ?)") _ROOM_COLUMNS = ( "room_id, name, members_json, authority_gateway_id, authority_epoch, next_seq, revision," - " created_at, updated_at, disbanded_at" -) + " created_at, updated_at, disbanded_at") _ROOM_COLUMNS_WITH_BYTES = ( "room_id, name, members_json, authority_gateway_id, authority_epoch, next_seq, event_bytes," - " revision, created_at, updated_at, disbanded_at" -) + " revision, created_at, updated_at, disbanded_at") _SELECT_ROOM = f"SELECT {_ROOM_COLUMNS} FROM hosted_rooms WHERE room_id=?" _SELECT_ROOM_WITH_BYTES = f"SELECT {_ROOM_COLUMNS_WITH_BYTES} FROM hosted_rooms WHERE room_id=?" _SUM_EVENT_BYTES = "SELECT COALESCE(SUM(event_bytes), 0) FROM hosted_rooms" -_INSERT_RETIRED = ( - "INSERT OR IGNORE INTO hosted_room_retired_ids (room_id, retired_at) VALUES (?, ?)" -) +_INSERT_RETIRED = ("INSERT OR IGNORE INTO hosted_room_retired_ids (room_id, retired_at) VALUES (?, ?)") _RETIRE_FROM_ROOMS = ( "INSERT OR IGNORE INTO hosted_room_retired_ids (room_id, retired_at)" - " SELECT room_id, disbanded_at FROM hosted_rooms WHERE {where}" -) + " SELECT room_id, disbanded_at FROM hosted_rooms WHERE {where}") +_LINK_COLUMNS = ( + "room_id", "member_id", "target_url", "target_profile", "grant", "catalog_json", + "cancellation_scope_id", "trace_id", "transport_security", "status", "updated_at") _REMOTE_RUN_WHERE = " AND ".join(f"{column}=?" for column in _REMOTE_RUN_IDENTITY_COLUMNS) -_LIVE_RESERVATION_WHERE = ( - "WHERE room_id=? AND target_profile=? AND expires_at>? AND revoked_at IS NULL" -) +_SELECT_REMOTE_RUN = f"SELECT * FROM hosted_room_remote_runs WHERE {_REMOTE_RUN_WHERE}" +_LIVE_RESERVATION_WHERE = ("WHERE room_id=? AND target_profile=? AND expires_at>? AND revoked_at IS NULL") _SELECT_LIVE_RESERVATION = ( - f"SELECT 1 FROM hosted_room_peer_reservations {_LIVE_RESERVATION_WHERE} LIMIT 1" -) + f"SELECT 1 FROM hosted_room_peer_reservations {_LIVE_RESERVATION_WHERE} LIMIT 1") -class HostedRoomError(ValueError): - """Base class for invalid or conflicting hosted-room operations.""" +class HostedRoomError(ValueError): """Base class for invalid or conflicting hosted-room operations.""" -class RoomNotFoundError(HostedRoomError): - """Raised when a room does not exist or has been disbanded.""" +class RoomNotFoundError(HostedRoomError): """Raised when a room does not exist or has been disbanded.""" class RoomHistoryExpiredError(RoomNotFoundError): @@ -241,16 +200,14 @@ class RoomHistoryExpiredError(RoomNotFoundError): reason = "room_history_expired" -class RoomConflictError(HostedRoomError): - """Raised when an idempotency key is reused for different room state.""" +class RoomConflictError(HostedRoomError): """Raised when an idempotency key is reused for different room state.""" class RoomProbeUnavailableError(HostedRoomError): """Raised when a non-blocking ownership probe cannot read the room store.""" -class EventConflictError(HostedRoomError): - """Raised when an event id is reused with different immutable content.""" +class EventConflictError(HostedRoomError): """Raised when an event id is reused with different immutable content.""" class AuthorityConflictError(HostedRoomError): @@ -265,23 +222,15 @@ class AuthoritySupersededError(AuthorityConflictError): # --- validation --------------------------------------------------------------- - -def _canonical_json(value: Any, *, label: str, max_bytes: int) -> str: - return canonical_json( - value, error=HostedRoomError, label=label, max_bytes=max_bytes, ensure_ascii=False - ) +_canonical_json = partial(canonical_json, error=HostedRoomError, ensure_ascii=False) +_validate_identifier = partial(identifier, error=HostedRoomError) +_room_id = partial(_validate_identifier, label="room_id", max_chars=MAX_ROOM_ID_CHARS) +_event_id = partial(_validate_identifier, label="event_id", max_chars=MAX_EVENT_ID_CHARS) +_actor_json = partial(_canonical_json, label="actor", max_bytes=4 * 1024) +_payload_json = partial(_canonical_json, label="payload", max_bytes=MAX_EVENT_JSON_BYTES) -def _validate_identifier(value: Any, *, label: str, max_chars: int) -> str: - return identifier(value, label=label, error=HostedRoomError, max_chars=max_chars) - - -def _room_id(value: Any) -> str: - return _validate_identifier(value, label="room_id", max_chars=MAX_ROOM_ID_CHARS) - - -def _event_id(value: Any, *, label: str = "event_id") -> str: - return _validate_identifier(value, label=label, max_chars=MAX_EVENT_ID_CHARS) +_bounded_int = partial(bounded_int, error=HostedRoomError) def _actor_id(value: Any, label: str) -> str: @@ -289,39 +238,24 @@ def _actor_id(value: Any, label: str) -> str: def _require_positive_int(value: Any, label: str) -> int: - return positive_int(value, error=HostedRoomError, message=f"{label} must be a positive integer") + return _bounded_int(value, message=f"{label} must be a positive integer", low=1) def _bounded_limit(value: Any, maximum: int) -> int: - if isinstance(value, bool) or not isinstance(value, int) or not 1 <= value <= maximum: - raise HostedRoomError(f"limit must be between 1 and {maximum}") - return value + return _bounded_int(value, message=f"limit must be between 1 and {maximum}", low=1, high=maximum) def _non_negative(value: Any, label: str) -> int: - return non_negative_int( - value, error=HostedRoomError, message=f"{label} must be a non-negative integer" - ) - - -def _now(now: float | None) -> float: - return time.time() if now is None else float(now) + return _bounded_int(value, message=f"{label} must be a non-negative integer") def _system_actor_json(actor_id: str) -> str: - return _canonical_json({"kind": "system", "id": actor_id}, label="actor", max_bytes=4 * 1024) - - -def _payload_json(payload: Any) -> str: - return _canonical_json(payload, label="payload", max_bytes=MAX_EVENT_JSON_BYTES) + return _actor_json({"kind": "system", "id": actor_id}) def _claim_payload_json(previous_gateway_id: str, new_gateway_id: str, epoch: int) -> str: - return _payload_json({ - "previous_gateway_id": previous_gateway_id, - "authority_gateway_id": new_gateway_id, - "authority_epoch": epoch, - }) + return _payload_json( + {"previous_gateway_id": previous_gateway_id, "authority_gateway_id": new_gateway_id, "authority_epoch": epoch}) def user_event_id(client_event_id: Any) -> str: @@ -330,13 +264,9 @@ def user_event_id(client_event_id: Any) -> str: return f"user:{hashlib.sha256(normalized.encode('utf-8')).hexdigest()}" -def _validate_room_name(value: Any) -> str: - if not isinstance(value, str): - raise HostedRoomError("name must be a string") - value = value.strip() - if not value or len(value) > MAX_ROOM_NAME_CHARS: - raise HostedRoomError("invalid room name") - return value +_validate_room_name = partial( + _validate_identifier, label="name", max_chars=MAX_ROOM_NAME_CHARS, pattern=None, + invalid="invalid room name") def _validate_members(value: Any) -> tuple[list[dict[str, Any]], str]: @@ -344,11 +274,9 @@ def _validate_members(value: Any) -> tuple[list[dict[str, Any]], str]: raise HostedRoomError("members must be a list") if len(value) > MAX_MEMBERS: raise HostedRoomError("too many room members") - members: list[dict[str, Any]] = [] - for member in value: - if not isinstance(member, dict): - raise HostedRoomError("each room member must be an object") - members.append(dict(member)) + if not all(isinstance(member, dict) for member in value): + raise HostedRoomError("each room member must be an object") + members = [dict(member) for member in value] return members, _canonical_json(members, label="members", max_bytes=MAX_MEMBERS_JSON_BYTES) @@ -373,23 +301,9 @@ def _legacy_members_match(existing_json: str, proposed: list[dict[str, Any]]) -> return True -def _validate_event_kind(value: Any) -> str: - return identifier( - value, label="kind", error=HostedRoomError, max_chars=MAX_EVENT_KIND_CHARS, - pattern=_EVENT_KIND_RE, invalid="invalid event kind", - ) - - -def _optional_actor_field(actor: dict[str, Any], field: str, max_chars: int) -> str: - value = actor.get(field) - if value is None: - return "" - if not isinstance(value, str): - raise HostedRoomError(f"actor.{field} must be a string") - value = value.strip() - if len(value) > max_chars: - raise HostedRoomError(f"actor.{field} is too long") - return value +_validate_event_kind = partial( + _validate_identifier, label="kind", max_chars=MAX_EVENT_KIND_CHARS, pattern=_EVENT_KIND_RE, + invalid="invalid event kind") def _validate_actor(value: Any, *, kind: str) -> tuple[dict[str, str], str]: @@ -404,15 +318,18 @@ def _validate_actor(value: Any, *, kind: str) -> tuple[dict[str, str], str]: if kind not in _EVENT_KINDS_BY_ACTOR[actor_kind]: raise HostedRoomError(f"actor kind '{actor_kind}' cannot append '{kind}'") actor = {"kind": actor_kind, "id": _actor_id(value.get("id"), "actor.id")} - for field, max_chars in ( - ("display_name", MAX_ACTOR_LABEL_CHARS), - ("profile", MAX_ACTOR_ID_CHARS), - ("connection_id", MAX_ACTOR_ID_CHARS), - ): - field_value = _optional_actor_field(value, field, max_chars) + for field, max_chars in _OPTIONAL_ACTOR_FIELDS: + field_value = value.get(field) + if field_value is None: + continue + if not isinstance(field_value, str): + raise HostedRoomError(f"actor.{field} must be a string") + field_value = field_value.strip() + if len(field_value) > max_chars: + raise HostedRoomError(f"actor.{field} is too long") if field_value: actor[field] = field_value - return actor, _canonical_json(actor, label="actor", max_bytes=4 * 1024) + return actor, _actor_json(actor) # --- schema / connections ------------------------------------------------------- @@ -426,8 +343,7 @@ def _primary_key_columns(conn: sqlite3.Connection, table: str) -> tuple[str, ... def _remote_run_schema_current(conn: sqlite3.Connection, columns: frozenset[str]) -> bool: return ( _REMOTE_RUN_SCHEMA_COLUMNS.issubset(columns) - and _primary_key_columns(conn, "hosted_room_remote_runs") == _REMOTE_RUN_IDENTITY_COLUMNS - ) + and _primary_key_columns(conn, "hosted_room_remote_runs") == _REMOTE_RUN_IDENTITY_COLUMNS) def _migrate_remote_run_schema(conn: sqlite3.Connection) -> None: @@ -451,8 +367,7 @@ def _migrate_remote_run_schema(conn: sqlite3.Connection) -> None: SELECT room_id, {home}, {gateway}, {epoch}, member_id, task_id, execution_generation, target_install_id, target_profile, run_id, session_id, created_at, updated_at - FROM hosted_room_remote_runs""" - ) + FROM hosted_room_remote_runs""") conn.execute("DROP TABLE hosted_room_remote_runs") conn.execute("ALTER TABLE hosted_room_remote_runs_migrating RENAME TO hosted_room_remote_runs") @@ -465,26 +380,16 @@ _LEGACY_ACTOR_JSON = _system_actor_json("legacy").replace("'", "''") _LEGACY_COLUMN_DDL = ( ( "hosted_rooms", "authority_gateway_id", - "ALTER TABLE hosted_rooms ADD COLUMN authority_gateway_id TEXT NOT NULL DEFAULT 'legacy'", - ), + "ALTER TABLE hosted_rooms ADD COLUMN authority_gateway_id TEXT NOT NULL DEFAULT 'legacy'"), ( "hosted_rooms", "authority_epoch", - "ALTER TABLE hosted_rooms ADD COLUMN authority_epoch INTEGER NOT NULL DEFAULT 1", - ), - ( - "hosted_rooms", "event_bytes", - "ALTER TABLE hosted_rooms ADD COLUMN event_bytes INTEGER NOT NULL DEFAULT 0", - ), + "ALTER TABLE hosted_rooms ADD COLUMN authority_epoch INTEGER NOT NULL DEFAULT 1"), + ("hosted_rooms", "event_bytes", "ALTER TABLE hosted_rooms ADD COLUMN event_bytes INTEGER NOT NULL DEFAULT 0"), ( "hosted_room_events", "actor_json", "ALTER TABLE hosted_room_events " - f"ADD COLUMN actor_json TEXT NOT NULL DEFAULT '{_LEGACY_ACTOR_JSON}'", - ), - ( - "hosted_room_events", "authority_epoch", - "ALTER TABLE hosted_room_events ADD COLUMN authority_epoch INTEGER", - ), -) + f"ADD COLUMN actor_json TEXT NOT NULL DEFAULT '{_LEGACY_ACTOR_JSON}'"), + ("hosted_room_events", "authority_epoch", "ALTER TABLE hosted_room_events ADD COLUMN authority_epoch INTEGER")) def _migrate_legacy_columns(conn: sqlite3.Connection) -> None: @@ -521,10 +426,7 @@ def _initialize_schema(conn: sqlite3.Connection) -> None: # permanent: a stale coordinate must never name a different Group Chat. conn.execute(_RETIRE_FROM_ROOMS.format(where="disbanded_at IS NOT NULL")) _migrate_remote_run_schema(conn) - conn.execute( - "CREATE INDEX IF NOT EXISTS idx_hosted_room_events_cursor " - "ON hosted_room_events(room_id, seq)" - ) + conn.execute("CREATE INDEX IF NOT EXISTS idx_hosted_room_events_cursor ON hosted_room_events(room_id, seq)") if not _schema_is_current(conn): raise HostedRoomError("hosted room schema migration did not complete") @@ -538,8 +440,7 @@ def _schema_is_current(conn: sqlite3.Connection) -> bool: if table == "hosted_room_remote_runs" and not _remote_run_schema_current(conn, columns): return False index = conn.execute( - "SELECT 1 FROM sqlite_master WHERE type='index' AND name='idx_hosted_room_events_cursor'" - ).fetchone() + "SELECT 1 FROM sqlite_master WHERE type='index' AND name='idx_hosted_room_events_cursor'").fetchone() return index is not None @@ -563,39 +464,9 @@ def local_authority_gateway_id() -> str: def _connect(db_path: Path | str) -> sqlite3.Connection: - from hermes_state import apply_wal_with_fallback - - path = Path(db_path) - path.parent.mkdir(parents=True, exist_ok=True) - conn = sqlite3.connect(path, timeout=10) - conn.row_factory = sqlite3.Row - try: - for attempt in range(_JOURNAL_MODE_LOCK_RETRIES): - try: - apply_wal_with_fallback(conn, db_label="state.db (hosted_rooms)") - break - except sqlite3.OperationalError as exc: - last = attempt + 1 == _JOURNAL_MODE_LOCK_RETRIES - if str(exc).lower() != "database is locked" or last: - raise - # SQLite's journal-mode pragma may not honor the connection's - # busy timeout while another first opener initializes the DB, - # especially on Windows. Retry only that transient lock class. - time.sleep(0.01 * (2**attempt)) - conn.execute("PRAGMA foreign_keys=ON") - if _schema_is_current(conn): - return conn - # Multiple profile gateways share this root database. Serialize every - # draft-schema transition in SQLite itself so a crash rolls back the - # whole DDL/data migration and another process can safely retry it. - conn.execute("BEGIN IMMEDIATE") - _initialize_schema(conn) - conn.commit() - except Exception: - conn.rollback() - conn.close() - raise - return conn + return connect( + db_path, db_label="state.db (hosted_rooms)", ready=_schema_is_current, + initialize=lambda conn: _initialize_schema(conn), lock_retries=_JOURNAL_MODE_LOCK_RETRIES) def _read_connection(db_path: Path | str) -> sqlite3.Connection: @@ -619,21 +490,31 @@ def _transaction(db_path: Path | str, *, immediate: bool = False): def _raise_room_not_found(conn: sqlite3.Connection, room_id: str) -> NoReturn: + # A retained disband tombstone still has replayable history; the caller simply + # did not opt into reading disbanded rooms. retained = conn.execute("SELECT 1 FROM hosted_rooms WHERE room_id=?", (room_id,)).fetchone() - if retained is not None: - # A retained disband tombstone still has replayable history. The - # caller simply did not opt into reading disbanded rooms. - raise RoomNotFoundError("hosted room not found") - retired = conn.execute( - "SELECT 1 FROM hosted_room_retired_ids WHERE room_id=?", (room_id,) - ).fetchone() - if retired is not None: - raise RoomHistoryExpiredError( - "Group Chat history expired; room_id remains permanently retired" - ) + if retained is None and _is_retired(conn, room_id): + raise RoomHistoryExpiredError("Group Chat history expired; room_id remains permanently retired") raise RoomNotFoundError("hosted room not found") +def _is_retired(conn: sqlite3.Connection, room_id: str) -> bool: + return conn.execute("SELECT 1 FROM hosted_room_retired_ids WHERE room_id=?", (room_id,)).fetchone() is not None + + +def _room_row(conn: sqlite3.Connection, sql: str, params: tuple[Any, ...], room_id: str) -> sqlite3.Row: + """Fetch one hosted_rooms row or raise the precise not-found/expired error.""" + row = conn.execute(sql, params).fetchone() + if row is None: + _raise_room_not_found(conn, room_id) + return row + + +def _require_authority(room: sqlite3.Row, gateway_id: str, epoch: int, message: str) -> None: + if str(room["authority_gateway_id"]) != gateway_id or int(room["authority_epoch"]) != epoch: + raise AuthorityConflictError(message) + + def _reload(conn: sqlite3.Connection, sql: str, params: tuple, missing: str) -> sqlite3.Row: """Re-read a row this transaction just wrote; a miss is an invariant violation.""" row = conn.execute(sql, params).fetchone() @@ -644,15 +525,10 @@ def _reload(conn: sqlite3.Connection, sql: str, params: tuple, missing: str) -> def _room_from_row(row: sqlite3.Row, *, idempotent: bool = False) -> dict[str, Any]: room = { - "room_id": row["room_id"], - "name": row["name"], - "members": json.loads(row["members_json"]), - "authority_gateway_id": row["authority_gateway_id"], - "authority_epoch": int(row["authority_epoch"]), - "revision": int(row["revision"]), - "created_at": float(row["created_at"]), - "updated_at": float(row["updated_at"]), - "idempotent": idempotent, + "room_id": row["room_id"], "name": row["name"], "members": json.loads(row["members_json"]), + "authority_gateway_id": row["authority_gateway_id"], "authority_epoch": int(row["authority_epoch"]), + "revision": int(row["revision"]), "created_at": float(row["created_at"]), + "updated_at": float(row["updated_at"]), "idempotent": idempotent, } # sqlite3.Row: ``x in row`` scans values, so ``.keys()`` is load-bearing. if "disbanded_at" in row.keys() and row["disbanded_at"] is not None: # noqa: SIM118 @@ -665,15 +541,9 @@ def _room_from_row(row: sqlite3.Row, *, idempotent: bool = False) -> dict[str, A def _event_from_row(row: sqlite3.Row, *, idempotent: bool = False) -> dict[str, Any]: epoch = row["authority_epoch"] return { - "room_id": row["room_id"], - "seq": int(row["seq"]), - "event_id": row["event_id"], - "kind": row["kind"], - "actor": json.loads(row["actor_json"]), - "authority_epoch": int(epoch) if epoch is not None else None, - "payload": json.loads(row["payload_json"]), - "created_at": float(row["created_at"]), - "idempotent": idempotent, + "room_id": row["room_id"], "seq": int(row["seq"]), "event_id": row["event_id"], "kind": row["kind"], + "actor": json.loads(row["actor_json"]), "authority_epoch": int(epoch) if epoch is not None else None, + "payload": json.loads(row["payload_json"]), "created_at": float(row["created_at"]), "idempotent": idempotent, } @@ -681,37 +551,43 @@ def _load_event(conn: sqlite3.Connection, room_id: str, event_id: str) -> sqlite return conn.execute(_SELECT_EVENT, (room_id, event_id)).fetchone() +def _event_content(row: sqlite3.Row) -> tuple[Any, Any, Any, Any]: + """The immutable (kind, actor_json, authority_epoch, payload_json) an event id is bound to.""" + return row["kind"], row["actor_json"], row["authority_epoch"], row["payload_json"] + + def _gateway_event_bytes(conn: sqlite3.Connection) -> int: return int(conn.execute(_SUM_EVENT_BYTES).fetchone()[0]) +def _insert_event( + conn: sqlite3.Connection, room: sqlite3.Row, room_id: str, seq: int, event_id: str, kind: str, + actor_json: str, epoch: int, payload_json: str, now: float, *, allow_control: bool = False) -> int: + """Capacity-check then INSERT one event at ``seq``; returns its accounted bytes.""" + event_bytes = _prepare_event(conn, room, event_id, kind, actor_json, payload_json, allow_control=allow_control) + conn.execute(_INSERT_EVENT, (room_id, seq, event_id, kind, actor_json, epoch, payload_json, now)) + return event_bytes + + def _prepare_event( conn: sqlite3.Connection, room: sqlite3.Row, event_id: str, kind: str, actor_json: str, - payload_json: str, *, allow_control: bool = False, -) -> int: + payload_json: str, *, allow_control: bool = False) -> int: """Size one pending event and enforce per-room and gateway capacity; returns its bytes.""" additional_bytes = len((event_id + kind + actor_json + payload_json).encode("utf-8")) count_reserve = CONTROL_EVENT_COUNT_RESERVE if allow_control else 0 byte_reserve = CONTROL_EVENT_BYTE_RESERVE if allow_control else 0 gateway_byte_limit = MAX_GATEWAY_EVENT_BYTES + byte_reserve if int(room["next_seq"]) - 1 >= MAX_EVENTS_PER_ROOM + count_reserve: - raise HostedRoomError( - "This Group Chat reached its history limit. Start a new Group Chat to continue." - ) + raise HostedRoomError("This Group Chat reached its history limit. Start a new Group Chat to continue.") if int(room["event_bytes"]) + additional_bytes > MAX_ROOM_EVENT_BYTES + byte_reserve: - raise HostedRoomError( - "This Group Chat reached its storage limit. Start a new Group Chat to continue." - ) + raise HostedRoomError("This Group Chat reached its storage limit. Start a new Group Chat to continue.") gateway_bytes = _gateway_event_bytes(conn) if gateway_bytes + additional_bytes > gateway_byte_limit: _prune_disbanded_rooms_locked( - conn, now=None, max_gateway_event_bytes=max(0, gateway_byte_limit - additional_bytes) - ) + conn, now=None, max_gateway_event_bytes=max(0, gateway_byte_limit - additional_bytes)) gateway_bytes = _gateway_event_bytes(conn) if gateway_bytes + additional_bytes > gateway_byte_limit: - raise HostedRoomError( - "Group Chat storage is full on this host. Delete an old Group Chat and try again." - ) + raise HostedRoomError("Group Chat storage is full on this host. Delete an old Group Chat and try again.") return additional_bytes @@ -719,33 +595,28 @@ def _prepare_event( # Deleted in this order when a disbanded room's payload is purged. _DEPENDENT_TABLES = ( - "hosted_room_policy_transcript_state", "hosted_room_policy_transcript", - "hosted_room_policy_publications", "hosted_room_policy_watermarks", - "hosted_room_policy_events", "hosted_room_policy_threads", "hosted_room_policy_cursors", - "hosted_room_driver_tasks", "hosted_room_driver_leases", "hosted_room_remote_runs", - "hosted_room_links", "hosted_room_peer_reservations", "hosted_room_events", -) + "hosted_room_policy_transcript_state", "hosted_room_policy_transcript", "hosted_room_policy_publications", + "hosted_room_policy_watermarks", "hosted_room_policy_events", "hosted_room_policy_threads", + "hosted_room_policy_cursors", "hosted_room_driver_tasks", "hosted_room_driver_leases", + "hosted_room_remote_runs", "hosted_room_links", "hosted_room_peer_reservations", "hosted_room_events") + + +def _room_ids(conn: sqlite3.Connection, sql: str, params: tuple[Any, ...]) -> list[str]: + return [str(row["room_id"]) for row in conn.execute(sql, params).fetchall()] def _prune_disbanded_rooms_locked( - conn: sqlite3.Connection, *, now: float | None, max_gateway_event_bytes: int | None = None -) -> int: + conn: sqlite3.Connection, *, now: float | None, max_gateway_event_bytes: int | None = None) -> int: candidates: set[str] = set() if now is not None: - candidates.update( - str(row["room_id"]) for row in conn.execute( - """SELECT room_id FROM hosted_rooms - WHERE disbanded_at IS NOT NULL AND disbanded_at<=?""", - (now - DISBANDED_ROOM_RETENTION_SECONDS,), - ).fetchall() - ) - candidates.update( - str(row["room_id"]) for row in conn.execute( - """SELECT room_id FROM hosted_rooms WHERE disbanded_at IS NOT NULL - ORDER BY disbanded_at DESC, room_id ASC LIMIT -1 OFFSET ?""", - (MAX_DISBANDED_ROOM_TOMBSTONES,), - ).fetchall() - ) + candidates.update(_room_ids( + conn, + """SELECT room_id FROM hosted_rooms + WHERE disbanded_at IS NOT NULL AND disbanded_at<=?""", (now - DISBANDED_ROOM_RETENTION_SECONDS,))) + candidates.update(_room_ids( + conn, + """SELECT room_id FROM hosted_rooms WHERE disbanded_at IS NOT NULL + ORDER BY disbanded_at DESC, room_id ASC LIMIT -1 OFFSET ?""", (MAX_DISBANDED_ROOM_TOMBSTONES,))) if max_gateway_event_bytes is not None: retained_bytes = _gateway_event_bytes(conn) if retained_bytes > max_gateway_event_bytes: @@ -763,10 +634,8 @@ def _prune_disbanded_rooms_locked( room_ids = tuple(sorted(candidates)) conn.execute( _RETIRE_FROM_ROOMS.format( - where=f"room_id IN ({placeholders}) AND disbanded_at IS NOT NULL" - ), - room_ids, - ) + where=f"room_id IN ({placeholders}) AND disbanded_at IS NOT NULL"), + room_ids) for table in _DEPENDENT_TABLES: if table_exists(conn, table): conn.execute(f"DELETE FROM {table} WHERE room_id IN ({placeholders})", room_ids) @@ -797,15 +666,12 @@ def list_room_link_records(db_path: Path | str) -> list[dict[str, Any]]: return [dict(row) for row in rows] -def upsert_room_link_record( - db_path: Path | str, *, record: Mapping[str, Any], max_links: int -) -> None: +def upsert_room_link_record(db_path: Path | str, *, record: Mapping[str, Any], max_links: int) -> None: """Atomically insert or replace one private RoomLink record.""" with _transaction(db_path, immediate=True) as conn: existing = conn.execute( "SELECT 1 FROM hosted_room_links WHERE room_id=? AND member_id=?", - (record["room_id"], record["member_id"]), - ).fetchone() + (record["room_id"], record["member_id"])).fetchone() if existing is None: count = int(conn.execute("SELECT COUNT(*) FROM hosted_room_links").fetchone()[0]) if count >= max_links: @@ -826,24 +692,16 @@ def upsert_room_link_record( transport_security=excluded.transport_security, status=excluded.status, updated_at=excluded.updated_at""", - ( - record["room_id"], record["member_id"], record["target_url"], - record["target_profile"], record["grant"], record["catalog_json"], - record["cancellation_scope_id"], record["trace_id"], - record["transport_security"], record["status"], record["updated_at"], - ), - ) + tuple(record[column] for column in _LINK_COLUMNS)) def update_room_link_status( - db_path: Path | str, *, room_id: str, member_id: str, status: str, now: float | None = None -) -> bool: + db_path: Path | str, *, room_id: str, member_id: str, status: str, now: float | None = None) -> bool: """Persist a non-secret route health classification.""" with _transaction(db_path, immediate=True) as conn: cursor = conn.execute( "UPDATE hosted_room_links SET status=?, updated_at=? WHERE room_id=? AND member_id=?", - (status, _now(now), room_id, member_id), - ) + (status, _now(now), room_id, member_id)) return cursor.rowcount == 1 @@ -859,19 +717,14 @@ def _room_grant_scope_key(claims: Mapping[str, Any]) -> str: key: str(claims.get(key) or "") for key in ( "room_id", "home_install_id", "authority_gateway_id", "authority_epoch", - "member_id", "target_install_id", "target_profile", - ) - } + "member_id", "target_install_id", "target_profile")} if not all(fields.values()): raise HostedRoomError("room grant scope is incomplete") - return hashlib.sha256( - json.dumps(fields, sort_keys=True, separators=(",", ":")).encode("utf-8") - ).hexdigest() + return hashlib.sha256(compact_json(fields).encode("utf-8")).hexdigest() def revoke_room_grant_scope( - db_path: Path | str, *, claims: Mapping[str, Any], expires_at: float, now: float | None = None -) -> None: + db_path: Path | str, *, claims: Mapping[str, Any], expires_at: float, now: float | None = None) -> None: """Revoke every grant issued at or before now for one exact room scope.""" scope_key = _room_grant_scope_key(claims) timestamp = _now(now) @@ -888,32 +741,24 @@ def revoke_room_grant_scope( expires_at=MAX(hosted_room_revoked_grants.expires_at, excluded.expires_at), revoked_before=MAX(hosted_room_revoked_grants.revoked_before, - excluded.revoked_before)""", - (scope_key, expiry, timestamp), - ) + excluded.revoked_before)""", (scope_key, expiry, timestamp)) conn.execute( """UPDATE hosted_room_peer_reservations SET revoked_at=?, updated_at=? WHERE room_id=? AND member_id=? AND target_profile=? AND authority_gateway_id=? AND authority_epoch=?""", ( - timestamp, timestamp, - str(claims.get("room_id") or ""), str(claims.get("member_id") or ""), - str(claims.get("target_profile") or ""), - str(claims.get("authority_gateway_id") or ""), + timestamp, timestamp, str(claims.get("room_id") or ""), str(claims.get("member_id") or ""), + str(claims.get("target_profile") or ""), str(claims.get("authority_gateway_id") or ""), int(claims.get("authority_epoch") or 0), - ), - ) + )) def _reservation_claims(claims: Mapping[str, Any]) -> tuple[str, str, str, str, int]: """Validate (room_id, member_id, target_profile, authority_gateway_id, authority_epoch).""" values = ( _room_id(claims.get("room_id")), - _actor_id(claims.get("member_id"), "member_id"), - _actor_id(claims.get("target_profile"), "target_profile"), - _actor_id(claims.get("authority_gateway_id"), "authority_gateway_id"), - int(claims.get("authority_epoch") or 0), - ) + *(_actor_id(claims.get(key), key) for key in ("member_id", "target_profile", "authority_gateway_id")), + int(claims.get("authority_epoch") or 0)) if values[4] < 1: raise HostedRoomError("authority_epoch must be positive") return values @@ -922,14 +767,11 @@ def _reservation_claims(claims: Mapping[str, Any]) -> tuple[str, str, str, str, def _reservation_superseded(row: sqlite3.Row, gateway_id: str, epoch: int) -> bool: """A newer epoch, or the same epoch under another gateway, outranks this claim.""" row_epoch = int(row["authority_epoch"]) - return row_epoch > epoch or ( - row_epoch == epoch and str(row["authority_gateway_id"]) != gateway_id - ) + return row_epoch > epoch or (row_epoch == epoch and str(row["authority_gateway_id"]) != gateway_id) def reserve_peer_room( - db_path: Path | str, *, claims: Mapping[str, Any], expires_at: float, now: float | None = None -) -> None: + db_path: Path | str, *, claims: Mapping[str, Any], expires_at: float, now: float | None = None) -> None: """Fence direct Desktop prompts before the first peer run is admitted.""" timestamp = _now(now) expiry = float(expires_at) @@ -942,20 +784,16 @@ def reserve_peer_room( authority_rows = conn.execute( f"""SELECT authority_gateway_id, authority_epoch FROM hosted_room_peer_reservations {_LIVE_RESERVATION_WHERE}""", - (room_id, target_profile, timestamp), - ).fetchall() + (room_id, target_profile, timestamp)).fetchall() if any(_reservation_superseded(row, gateway_id, epoch) for row in authority_rows): raise AuthorityConflictError("peer room reservation authority changed") conn.execute( """UPDATE hosted_room_peer_reservations SET revoked_at=?, updated_at=? WHERE room_id=? AND target_profile=? AND authority_epoch sqlite3.Row | None: @@ -979,18 +815,14 @@ def _read_one(db_path: Path | str, sql: str, params: tuple[Any, ...]) -> sqlite3 return conn.execute(sql, params).fetchone() -def peer_room_is_reserved( - db_path: Path | str, *, room_id: str, target_profile: str, now: float | None = None -) -> bool: +def peer_room_is_reserved(db_path: Path | str, *, room_id: str, target_profile: str, now: float | None = None) -> bool: """Return whether a live target-side RoomLink reservation fences Desktop.""" timestamp = _now(now) params = (_room_id(room_id), _actor_id(target_profile, "target_profile"), timestamp) return _read_one(db_path, _SELECT_LIVE_RESERVATION, params) is not None -def peer_room_grant_is_current( - db_path: Path | str, *, claims: Mapping[str, Any], now: float | None = None -) -> bool: +def peer_room_grant_is_current(db_path: Path | str, *, claims: Mapping[str, Any], now: float | None = None) -> bool: """Require a grant to match the target's current live reservation.""" timestamp = _now(now) values = _reservation_claims(claims) @@ -998,15 +830,11 @@ def peer_room_grant_is_current( db_path, """SELECT 1 FROM hosted_room_peer_reservations WHERE room_id=? AND member_id=? AND target_profile=? AND authority_gateway_id=? AND authority_epoch=? - AND expires_at>? AND revoked_at IS NULL LIMIT 1""", - (*values, timestamp), - ) + AND expires_at>? AND revoked_at IS NULL LIMIT 1""", (*values, timestamp)) return row is not None -def room_grant_is_revoked( - db_path: Path | str, *, claims: Mapping[str, Any], now: float | None = None -) -> bool: +def room_grant_is_revoked(db_path: Path | str, *, claims: Mapping[str, Any], now: float | None = None) -> bool: """Return whether a grant predates its exact scope's revocation fence.""" timestamp = _now(now) scope_key = _room_grant_scope_key(claims) @@ -1014,9 +842,7 @@ def room_grant_is_revoked( row = _read_one( db_path, """SELECT revoked_before FROM hosted_room_revoked_grants - WHERE scope_key=? AND expires_at>?""", - (scope_key, timestamp), - ) + WHERE scope_key=? AND expires_at>?""", (scope_key, timestamp)) return row is not None and issued_at <= float(row["revoked_before"]) @@ -1024,25 +850,19 @@ def _remote_run_identity(record: Mapping[str, Any]) -> tuple[Any, ...]: return tuple(record[column] for column in _REMOTE_RUN_IDENTITY_COLUMNS) -def upsert_remote_run_receipt( - db_path: Path | str, *, record: Mapping[str, Any], now: float | None = None -) -> None: +def upsert_remote_run_receipt(db_path: Path | str, *, record: Mapping[str, Any], now: float | None = None) -> None: """Durably bind one logical peer task attempt to its remote run handle.""" timestamp = _now(now) identity = _remote_run_identity(record) + immutable = (*identity, record["run_id"], record["session_id"]) with _transaction(db_path, immediate=True) as conn: - existing = conn.execute( - f"SELECT * FROM hosted_room_remote_runs WHERE {_REMOTE_RUN_WHERE}", identity - ).fetchone() - immutable = (*identity, record["run_id"], record["session_id"]) + existing = conn.execute(_SELECT_REMOTE_RUN, identity).fetchone() if existing is not None: - stored = (*_remote_run_identity(existing), existing["run_id"], existing["session_id"]) - if stored != immutable: + if (*_remote_run_identity(existing), existing["run_id"], existing["session_id"]) != immutable: raise HostedRoomError("remote run receipt conflicts with its logical task") conn.execute( f"UPDATE hosted_room_remote_runs SET updated_at=? WHERE {_REMOTE_RUN_WHERE}", - (timestamp, *identity), - ) + (timestamp, *identity)) return conn.execute( """INSERT INTO hosted_room_remote_runs( @@ -1050,40 +870,29 @@ def upsert_remote_run_receipt( authority_epoch, member_id, target_install_id, target_profile, task_id, execution_generation, run_id, session_id, created_at, updated_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", - (*immutable, timestamp, timestamp), - ) + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", (*immutable, timestamp, timestamp)) def list_remote_run_receipts( db_path: Path | str, *, room_id: str | None = None, target_profile: str | None = None, - session_id: str | None = None, -) -> list[dict[str, Any]]: + session_id: str | None = None) -> list[dict[str, Any]]: """Return remote run handles in durable task order.""" filters = [ (column, value) - for column, value in ( - ("room_id", room_id), ("target_profile", target_profile), ("session_id", session_id) - ) - if value is not None - ] + for column, value in (("room_id", room_id), ("target_profile", target_profile), ("session_id", session_id)) + if value is not None] where = f" WHERE {' AND '.join(f'{column}=?' for column, _ in filters)}" if filters else "" with _transaction(db_path) as conn: rows = conn.execute( "SELECT * FROM hosted_room_remote_runs" + where + " ORDER BY created_at, task_id, execution_generation", - [value for _, value in filters], - ).fetchall() + [value for _, value in filters]).fetchall() return [dict(row) for row in rows] def remote_run_receipt(db_path: Path | str, *, record: Mapping[str, Any]) -> dict[str, Any] | None: """Return the exact durable remote run handle for one task attempt.""" - row = _read_one( - db_path, - f"SELECT * FROM hosted_room_remote_runs WHERE {_REMOTE_RUN_WHERE}", - _remote_run_identity(record), - ) + row = _read_one(db_path, _SELECT_REMOTE_RUN, _remote_run_identity(record)) return dict(row) if row is not None else None @@ -1092,24 +901,15 @@ def remote_run_receipt(db_path: Path | str, *, record: Mapping[str, Any]) -> dic def _adopt_legacy_room( conn: sqlite3.Connection, existing: sqlite3.Row, *, room_id: str, members_json: str, - authority_gateway_id: str, now: float, -) -> dict[str, Any]: + authority_gateway_id: str, now: float) -> dict[str, Any]: """Claim a 'legacy'-authority room for a real gateway with a fenced claim event.""" target_epoch = int(existing["authority_epoch"]) + 1 seq = int(existing["next_seq"]) actor_json = _system_actor_json("authority-control") payload_json = _claim_payload_json("legacy", authority_gateway_id, target_epoch) - claim_bytes = _prepare_event( - conn, existing, "system:authority-adopted", "authority.claimed", actor_json, - payload_json, allow_control=True, - ) - conn.execute( - _INSERT_EVENT, - ( - room_id, seq, "system:authority-adopted", "authority.claimed", actor_json, - target_epoch, payload_json, now, - ), - ) + claim_bytes = _insert_event( + conn, existing, room_id, seq, "system:authority-adopted", "authority.claimed", actor_json, + target_epoch, payload_json, now, allow_control=True) adopted = conn.execute( """UPDATE hosted_rooms SET members_json=?, authority_gateway_id=?, authority_epoch=?, @@ -1118,9 +918,7 @@ def _adopt_legacy_room( AND disbanded_at IS NULL""", ( members_json, authority_gateway_id, target_epoch, claim_bytes, now, room_id, - int(existing["authority_epoch"]), seq, - ), - ) + int(existing["authority_epoch"]), seq)) if adopted.rowcount != 1: raise AuthorityConflictError("legacy room adoption lost its fence") existing = _reload(conn, _SELECT_ROOM, (room_id,), "adopted room could not be reloaded") @@ -1128,16 +926,14 @@ def _adopt_legacy_room( result["adopted"] = True claim_event = _reload( conn, _SELECT_EVENT, (room_id, "system:authority-adopted"), - "legacy adoption event could not be reloaded", - ) + "legacy adoption event could not be reloaded") result["claim_event"] = _event_from_row(claim_event) return result def create_room( db_path: Path | str, *, room_id: Any, name: Any, members: Any, authority_gateway_id: Any, - now: float | None = None, -) -> dict[str, Any]: + now: float | None = None) -> dict[str, Any]: """Create a room, or return the identical existing room idempotently.""" room_id = _room_id(room_id) name = _validate_room_name(name) @@ -1146,61 +942,45 @@ def create_room( now = _now(now) with _transaction(db_path, immediate=True) as conn: - if conn.execute( - "SELECT 1 FROM hosted_room_retired_ids WHERE room_id=?", (room_id,) - ).fetchone(): + if _is_retired(conn, room_id): raise RoomConflictError("room_id belongs to a disbanded room") existing = conn.execute(_SELECT_ROOM_WITH_BYTES, (room_id,)).fetchone() if existing is not None: if existing["disbanded_at"] is not None: raise RoomConflictError("room_id belongs to a disbanded room") - legacy_adoption = ( - existing["authority_gateway_id"] == "legacy" and authority_gateway_id != "legacy" - ) + legacy_adoption = (existing["authority_gateway_id"] == "legacy" and authority_gateway_id != "legacy") members_match = existing["members_json"] == members_json or ( legacy_adoption - and _legacy_members_match(existing["members_json"], normalized_members) - ) + and _legacy_members_match(existing["members_json"], normalized_members)) if existing["name"] != name or not members_match: raise RoomConflictError("room_id already exists with different state") if legacy_adoption: return _adopt_legacy_room( conn, existing, room_id=room_id, members_json=members_json, - authority_gateway_id=authority_gateway_id, now=now, - ) + authority_gateway_id=authority_gateway_id, now=now) if existing["authority_gateway_id"] != authority_gateway_id: raise RoomConflictError("room_id already belongs to a different authority") return _room_from_row(existing, idempotent=True) - active_rooms = int( - conn.execute( - "SELECT COUNT(*) FROM hosted_rooms WHERE disbanded_at IS NULL" - ).fetchone()[0] - ) + active_rooms = int(conn.execute("SELECT COUNT(*) FROM hosted_rooms WHERE disbanded_at IS NULL").fetchone()[0]) if active_rooms >= MAX_ACTIVE_ROOMS: - raise HostedRoomError( - "This host has too many active Group Chats. Delete one and try again." - ) + raise HostedRoomError("This host has too many active Group Chats. Delete one and try again.") conn.execute( f"""INSERT INTO hosted_rooms ({_ROOM_COLUMNS_WITH_BYTES}) VALUES (?, ?, ?, ?, 1, 1, 0, 1, ?, ?, NULL)""", - (room_id, name, members_json, authority_gateway_id, now, now), - ) + (room_id, name, members_json, authority_gateway_id, now, now)) row = _reload( conn, """SELECT room_id, name, members_json, authority_gateway_id, authority_epoch, revision, - created_at, updated_at FROM hosted_rooms WHERE room_id=?""", - (room_id,), - "created room could not be reloaded", - ) + created_at, updated_at FROM hosted_rooms WHERE room_id=?""", (room_id,), + "created room could not be reloaded") result = _room_from_row(row) result["members"] = normalized_members return result def list_rooms( - db_path: Path | str, *, include_disbanded: bool = False, limit: int = MAX_ROOM_LIST_LIMIT, - offset: int = 0, + db_path: Path | str, *, include_disbanded: bool = False, limit: int = MAX_ROOM_LIST_LIMIT, offset: int = 0, ) -> list[dict[str, Any]]: """Return one bounded read-only page ordered by most recent change.""" limit = _bounded_limit(limit, MAX_ROOM_LIST_LIMIT) @@ -1210,16 +990,14 @@ def list_rooms( rows = conn.execute( f"""SELECT {_ROOM_COLUMNS} FROM hosted_rooms WHERE disbanded_at IS NULL OR ? ORDER BY updated_at DESC, room_id ASC LIMIT ? OFFSET ?""", - (int(include_disbanded), limit, offset), - ).fetchall() + (int(include_disbanded), limit, offset)).fetchall() finally: conn.close() return [_room_from_row(row) for row in rows] def rename_room( - db_path: Path | str, *, room_id: Any, event_id: Any, name: Any, now: float | None = None -) -> dict[str, Any]: + db_path: Path | str, *, room_id: Any, event_id: Any, name: Any, now: float | None = None) -> dict[str, Any]: """Rename a live room and append its replay event atomically.""" room_id = _room_id(room_id) event_id = _event_id(event_id) @@ -1228,9 +1006,7 @@ def rename_room( actor_json = _system_actor_json("room-control") payload_json = _payload_json({"name": name}) with _transaction(db_path, immediate=True) as conn: - room = conn.execute(_SELECT_ROOM_WITH_BYTES, (room_id,)).fetchone() - if room is None: - _raise_room_not_found(conn, room_id) + room = _room_row(conn, _SELECT_ROOM_WITH_BYTES, (room_id,), room_id) if room["disbanded_at"] is not None: raise RoomNotFoundError("hosted room not found") existing = _load_event(conn, room_id, event_id) @@ -1247,13 +1023,8 @@ def rename_room( conn.execute( """UPDATE hosted_rooms SET name=?, next_seq=?, event_bytes=event_bytes+?, revision=revision+1, updated_at=? - WHERE room_id=?""", - (name, seq + 1, event_bytes, now, room_id), - ) - conn.execute( - _INSERT_EVENT, - (room_id, seq, event_id, "room.renamed", actor_json, epoch, payload_json, now), - ) + WHERE room_id=?""", (name, seq + 1, event_bytes, now, room_id)) + conn.execute(_INSERT_EVENT, (room_id, seq, event_id, "room.renamed", actor_json, epoch, payload_json, now)) updated = conn.execute(_SELECT_ROOM, (room_id,)).fetchone() event = _load_event(conn, room_id, event_id) result = _room_from_row(updated) @@ -1263,12 +1034,10 @@ def rename_room( def append_event( db_path: Path | str, *, room_id: Any, event_id: Any, kind: Any, actor: Any, payload: Any, - authority_gateway_id: Any = None, authority_epoch: Any = None, now: float | None = None, -) -> dict[str, Any]: + authority_gateway_id: Any = None, authority_epoch: Any = None, now: float | None = None) -> dict[str, Any]: """Append one immutable event and allocate its per-room sequence atomically. - Repeating the same ``event_id`` and immutable content returns the original - event. Reusing the id for different content fails closed. + Repeating an ``event_id`` with identical content returns the original; different content fails closed. """ room_id = _room_id(room_id) event_id = _event_id(event_id) @@ -1287,48 +1056,28 @@ def append_event( with _transaction(db_path, immediate=True) as conn: existing = _load_event(conn, room_id, event_id) if existing is not None: - if ( - existing["kind"] != kind - or existing["actor_json"] != actor_json - or existing["authority_epoch"] != authority_epoch - or existing["payload_json"] != payload_json - ): + if _event_content(existing) != (kind, actor_json, authority_epoch, payload_json): raise EventConflictError("event_id already exists with different content") return _event_from_row(existing, idempotent=True) - room = conn.execute( + room = _room_row( + conn, """SELECT next_seq, event_bytes, authority_gateway_id, authority_epoch - FROM hosted_rooms WHERE room_id=? AND disbanded_at IS NULL""", - (room_id,), - ).fetchone() - if room is None: - _raise_room_not_found(conn, room_id) - if ( - room["authority_gateway_id"] != authority_gateway_id - or int(room["authority_epoch"]) != authority_epoch - ): - raise AuthorityConflictError("stale hosted room authority") + FROM hosted_rooms WHERE room_id=? AND disbanded_at IS NULL""", (room_id,), room_id) + _require_authority(room, authority_gateway_id, authority_epoch, "stale hosted room authority") seq = int(room["next_seq"]) - event_bytes = _prepare_event( - conn, room, event_id, kind, actor_json, payload_json, - allow_control=kind in _CONTROL_EVENT_KINDS, - ) - conn.execute( - _INSERT_EVENT, - (room_id, seq, event_id, kind, actor_json, authority_epoch, payload_json, now), - ) + event_bytes = _insert_event( + conn, room, room_id, seq, event_id, kind, actor_json, authority_epoch, payload_json, now, + allow_control=kind in _CONTROL_EVENT_KINDS) advanced = conn.execute( """UPDATE hosted_rooms SET next_seq=?, event_bytes=event_bytes+?, updated_at=? - WHERE room_id=? AND next_seq=?""", - (seq + 1, event_bytes, now, room_id, seq), - ) + WHERE room_id=? AND next_seq=?""", (seq + 1, event_bytes, now, room_id, seq)) if advanced.rowcount != 1: raise RuntimeError("hosted room sequence advance lost its write fence") row = _reload( conn, f"SELECT {_EVENT_COLUMNS} FROM hosted_room_events WHERE room_id=? AND seq=?", (room_id, seq), - "appended event could not be reloaded", - ) + "appended event could not be reloaded") result = _event_from_row(row) result["actor"] = normalized_actor return result @@ -1342,11 +1091,8 @@ def _probe(path: Path, table: str, query: str, params: tuple[Any, ...], unavaila conn = sqlite3.connect(path, timeout=0.05) try: table_row = conn.execute( - f"SELECT 1 FROM sqlite_master WHERE type='table' AND name='{table}' LIMIT 1" - ).fetchone() - if table_row is None: - return False - return conn.execute(query, params).fetchone() is not None + f"SELECT 1 FROM sqlite_master WHERE type='table' AND name='{table}' LIMIT 1").fetchone() + return table_row is not None and conn.execute(query, params).fetchone() is not None finally: conn.close() except sqlite3.Error as exc: @@ -1356,94 +1102,71 @@ def _probe(path: Path, table: str, query: str, params: tuple[Any, ...], unavaila def probe_hosted_room(db_path: Path | str, *, room_id: Any) -> bool: """Check room ownership without creating or migrating the shared store. - This runs on the synchronous prompt-admission path for older Desktop - clients, so it fails quickly under contention instead of blocking the - WebSocket reader for SQLite's normal ten-second timeout. + Runs on the synchronous prompt-admission path for older Desktop clients, so it fails fast + under contention instead of blocking the WebSocket reader for SQLite's ten-second timeout. """ checked_room_id = _room_id(room_id) return _probe( Path(db_path), "hosted_rooms", "SELECT 1 FROM hosted_rooms WHERE room_id=? AND disbanded_at IS NULL LIMIT 1", - (checked_room_id,), "hosted room ownership is temporarily unavailable", - ) + (checked_room_id,), "hosted room ownership is temporarily unavailable") def probe_peer_room_reservation( - db_path: Path | str, *, room_id: Any, target_profile: Any, now: float | None = None -) -> bool: + db_path: Path | str, *, room_id: Any, target_profile: Any, now: float | None = None) -> bool: """Check a peer reservation without creating or migrating shared state.""" checked_room_id = _room_id(room_id) checked_profile = _actor_id(target_profile, "target_profile") - path = Path(db_path) - if not path.is_file(): - return False return _probe( - path, "hosted_room_peer_reservations", _SELECT_LIVE_RESERVATION, - (checked_room_id, checked_profile, _now(now)), - "peer room ownership is temporarily unavailable", - ) + Path(db_path), "hosted_room_peer_reservations", _SELECT_LIVE_RESERVATION, + (checked_room_id, checked_profile, _now(now)), "peer room ownership is temporarily unavailable") -def room_state( - db_path: Path | str, *, room_id: Any, include_disbanded: bool = False -) -> dict[str, Any]: +def room_state(db_path: Path | str, *, room_id: Any, include_disbanded: bool = False) -> dict[str, Any]: """Return durable replay and authority state for one room.""" room_id = _room_id(room_id) with _transaction(db_path) as conn: - row = conn.execute( + row = _room_row( + conn, f"""SELECT {_ROOM_COLUMNS} FROM hosted_rooms WHERE room_id=? AND (disbanded_at IS NULL OR ?)""", - (room_id, int(include_disbanded)), - ).fetchone() - if row is None: - _raise_room_not_found(conn, room_id) + (room_id, int(include_disbanded)), room_id) claim_row = conn.execute( f"""SELECT {_EVENT_COLUMNS} FROM hosted_room_events WHERE room_id=? AND kind='authority.claimed' AND authority_epoch=? ORDER BY seq DESC LIMIT 1""", - (room_id, int(row["authority_epoch"])), - ).fetchone() + (room_id, int(row["authority_epoch"]))).fetchone() state = _room_from_row(row) - state["latest_seq"] = int(row["next_seq"]) - 1 if claim_row is not None: state["authority_claim"] = _event_from_row(claim_row) return state def request_room_stop( - db_path: Path | str, *, room_id: Any, cancel_id: Any, expected_gateway_id: Any, - expected_epoch: Any, + db_path: Path | str, *, room_id: Any, cancel_id: Any, expected_gateway_id: Any, expected_epoch: Any, ) -> dict[str, Any]: """Append an idempotent fence that supersedes earlier user turns.""" - cancel_id = _event_id(cancel_id, label="cancel_id") + cancel_id = _validate_identifier(cancel_id, label="cancel_id", max_chars=MAX_EVENT_ID_CHARS) digest = hashlib.sha256(cancel_id.encode()).hexdigest()[:32] return append_event( db_path, room_id=room_id, event_id=f"room-stop:{digest}", kind="room.stop_requested", actor={"kind": "gateway", "id": expected_gateway_id}, payload={"cancel_id": cancel_id}, - authority_gateway_id=expected_gateway_id, authority_epoch=expected_epoch, - ) + authority_gateway_id=expected_gateway_id, authority_epoch=expected_epoch) def _append_authority_claim( conn: sqlite3.Connection, row: sqlite3.Row, *, room_id: str, event_id: str, expected_gateway_id: str, expected_epoch: int, new_gateway_id: str, target_epoch: int, - actor_json: str, payload_json: str, now: float, -) -> sqlite3.Row | None: + actor_json: str, payload_json: str, now: float) -> sqlite3.Row | None: """Insert the claim event and CAS the room's authority; returns the stored claim event.""" - seq = int(row["next_seq"]) - claim_bytes = _prepare_event( - conn, row, event_id, "authority.claimed", actor_json, payload_json, allow_control=True - ) - conn.execute( - _INSERT_EVENT, - (room_id, seq, event_id, "authority.claimed", actor_json, target_epoch, payload_json, now), - ) + claim_bytes = _insert_event( + conn, row, room_id, int(row["next_seq"]), event_id, "authority.claimed", actor_json, + target_epoch, payload_json, now, allow_control=True) updated = conn.execute( """UPDATE hosted_rooms SET authority_gateway_id=?, authority_epoch=authority_epoch+1, next_seq=next_seq+1, event_bytes=event_bytes+?, revision=revision+1, updated_at=? WHERE room_id=? AND disbanded_at IS NULL AND authority_gateway_id=? AND authority_epoch=?""", - (new_gateway_id, claim_bytes, now, room_id, expected_gateway_id, expected_epoch), - ) + (new_gateway_id, claim_bytes, now, room_id, expected_gateway_id, expected_epoch)) if updated.rowcount != 1: raise AuthorityConflictError("hosted room authority changed") return _load_event(conn, room_id, event_id) @@ -1451,13 +1174,11 @@ def _append_authority_claim( def claim_authority( db_path: Path | str, *, room_id: Any, expected_gateway_id: Any, expected_epoch: Any, - new_gateway_id: Any, event_id: Any, now: float | None = None, -) -> dict[str, Any]: + new_gateway_id: Any, event_id: Any, now: float | None = None) -> dict[str, Any]: """Fence a verified authority transfer with a compare-and-swap epoch. - This storage primitive does not decide *when* takeover is safe. A - replicated driver must call it only after its lease/quorum policy has - established that the previous owner can no longer commit. + Does not decide *when* takeover is safe: a replicated driver calls it only after its + lease/quorum policy established that the previous owner can no longer commit. """ room_id = _room_id(room_id) expected_gateway_id = _actor_id(expected_gateway_id, "expected_gateway_id") @@ -1470,65 +1191,47 @@ def claim_authority( claim_payload_json = _claim_payload_json(expected_gateway_id, new_gateway_id, target_epoch) with _transaction(db_path, immediate=True) as conn: - row = conn.execute( + row = _room_row( + conn, """SELECT authority_gateway_id, authority_epoch, next_seq, event_bytes - FROM hosted_rooms WHERE room_id=? AND disbanded_at IS NULL""", - (room_id,), - ).fetchone() - if row is None: - _raise_room_not_found(conn, room_id) + FROM hosted_rooms WHERE room_id=? AND disbanded_at IS NULL""", (room_id,), room_id) current_gateway = str(row["authority_gateway_id"]) current_epoch = int(row["authority_epoch"]) existing_event = _load_event(conn, room_id, event_id) idempotent = existing_event is not None if idempotent: - if ( - existing_event["kind"] != "authority.claimed" - or existing_event["actor_json"] != claim_actor_json - or existing_event["authority_epoch"] != target_epoch - or existing_event["payload_json"] != claim_payload_json - ): + if _event_content(existing_event) != ( + "authority.claimed", claim_actor_json, target_epoch, claim_payload_json): raise EventConflictError("event_id already exists with different content") if current_gateway != new_gateway_id or current_epoch != target_epoch: raise AuthoritySupersededError("authority claim succeeded but was later superseded") - elif current_gateway != expected_gateway_id or current_epoch != expected_epoch: - raise AuthorityConflictError("hosted room authority changed") else: + _require_authority(row, expected_gateway_id, expected_epoch, "hosted room authority changed") existing_event = _append_authority_claim( - conn, row, room_id=room_id, event_id=event_id, - expected_gateway_id=expected_gateway_id, expected_epoch=expected_epoch, - new_gateway_id=new_gateway_id, target_epoch=target_epoch, - actor_json=claim_actor_json, payload_json=claim_payload_json, now=now, - ) + conn, row, room_id=room_id, event_id=event_id, expected_gateway_id=expected_gateway_id, + expected_epoch=expected_epoch, new_gateway_id=new_gateway_id, target_epoch=target_epoch, + actor_json=claim_actor_json, payload_json=claim_payload_json, now=now) state_row = _reload( conn, """SELECT room_id, name, members_json, authority_gateway_id, authority_epoch, next_seq, - revision, created_at, updated_at FROM hosted_rooms WHERE room_id=?""", - (room_id,), - "claimed room could not be reloaded", - ) + revision, created_at, updated_at FROM hosted_rooms WHERE room_id=?""", (room_id,), + "claimed room could not be reloaded") state = _room_from_row(state_row, idempotent=idempotent) - state["latest_seq"] = int(state_row["next_seq"]) - 1 if existing_event is None: # pragma: no cover - both claim paths set it raise RuntimeError("authority claim event could not be reloaded") state["claim_event"] = _event_from_row(existing_event, idempotent=idempotent) return state -def _disband_replay( - conn: sqlite3.Connection, room_id: str, room: sqlite3.Row | None -) -> dict[str, Any] | None: +def _disband_replay(conn: sqlite3.Connection, room_id: str, room: sqlite3.Row | None) -> dict[str, Any] | None: """Idempotent replay for a retired or already-disbanded room; None when the room is live.""" if room is None: - retired = conn.execute( - "SELECT retired_at FROM hosted_room_retired_ids WHERE room_id=?", (room_id,) - ).fetchone() + retired = conn.execute("SELECT retired_at FROM hosted_room_retired_ids WHERE room_id=?", (room_id,)).fetchone() if retired is None: raise RoomNotFoundError("hosted room not found") return { "room_id": room_id, "disbanded_at": float(retired["retired_at"]), - "idempotent": True, "history_expired": True, - } + "idempotent": True, "history_expired": True} if room["disbanded_at"] is None: return None conn.execute(_INSERT_RETIRED, (room_id, float(room["disbanded_at"]))) @@ -1541,8 +1244,7 @@ def _disband_replay( def disband_room( db_path: Path | str, *, room_id: Any, expected_gateway_id: Any, expected_epoch: Any, - now: float | None = None, -) -> dict[str, Any]: + now: float | None = None) -> dict[str, Any]: """Tombstone a room id permanently and idempotently.""" room_id = _room_id(room_id) expected_gateway_id = _actor_id(expected_gateway_id, "expected_gateway_id") @@ -1552,58 +1254,34 @@ def disband_room( with _transaction(db_path, immediate=True) as conn: room = conn.execute( """SELECT authority_gateway_id, authority_epoch, next_seq, event_bytes, disbanded_at - FROM hosted_rooms WHERE room_id=?""", - (room_id,), - ).fetchone() + FROM hosted_rooms WHERE room_id=?""", (room_id,)).fetchone() replay = _disband_replay(conn, room_id, room) if replay is not None: return replay - if ( - str(room["authority_gateway_id"]) != expected_gateway_id - or int(room["authority_epoch"]) != expected_epoch - ): - raise AuthorityConflictError("stale hosted room authority") - seq = int(room["next_seq"]) - actor_json = _system_actor_json("room-control") - payload_json = _payload_json({"room_id": room_id}) - disband_bytes = _prepare_event( - conn, room, "system:room-disbanded", "room.disbanded", actor_json, payload_json, - allow_control=True, - ) - conn.execute( - _INSERT_EVENT, - ( - room_id, seq, "system:room-disbanded", "room.disbanded", actor_json, - int(room["authority_epoch"]), payload_json, now, - ), - ) + _require_authority(room, expected_gateway_id, expected_epoch, "stale hosted room authority") + disband_bytes = _insert_event( + conn, room, room_id, int(room["next_seq"]), "system:room-disbanded", "room.disbanded", + _system_actor_json("room-control"), int(room["authority_epoch"]), + _payload_json({"room_id": room_id}), now, allow_control=True) updated = conn.execute( """UPDATE hosted_rooms SET disbanded_at=?, updated_at=?, revision=revision+1, next_seq=next_seq+1, event_bytes=event_bytes+? WHERE room_id=? AND disbanded_at IS NULL AND authority_gateway_id=? AND authority_epoch=?""", - (now, now, disband_bytes, room_id, expected_gateway_id, expected_epoch), - ) + (now, now, disband_bytes, room_id, expected_gateway_id, expected_epoch)) if updated.rowcount != 1: raise RoomConflictError("hosted room disband lost its fence") conn.execute(_INSERT_RETIRED, (room_id, now)) event = _reload( conn, _SELECT_EVENT, (room_id, "system:room-disbanded"), - "room disband event could not be reloaded", - ) - _prune_disbanded_rooms_locked( - conn, now=now, max_gateway_event_bytes=MAX_GATEWAY_EVENT_BYTES - ) - return { - "room_id": room_id, "disbanded_at": now, "idempotent": False, - "event": _event_from_row(event), - } + "room disband event could not be reloaded") + _prune_disbanded_rooms_locked(conn, now=now, max_gateway_event_bytes=MAX_GATEWAY_EVENT_BYTES) + return {"room_id": room_id, "disbanded_at": now, "idempotent": False, "event": _event_from_row(event)} def read_events( - db_path: Path | str, *, room_id: Any, since_seq: Any = 0, limit: Any = 100, - include_disbanded: bool = False, + db_path: Path | str, *, room_id: Any, since_seq: Any = 0, limit: Any = 100, include_disbanded: bool = False, ) -> dict[str, Any]: """Read a monotonic room-log delta after ``since_seq``.""" room_id = _room_id(room_id) @@ -1611,17 +1289,12 @@ def read_events( limit = _bounded_limit(limit, MAX_LOG_LIMIT) with _transaction(db_path) as conn: - room = conn.execute( + room = _room_row( + conn, """SELECT next_seq, authority_gateway_id, authority_epoch FROM hosted_rooms - WHERE room_id=? AND (disbanded_at IS NULL OR ?)""", - (room_id, int(include_disbanded)), - ).fetchone() - if room is None: - _raise_room_not_found(conn, room_id) + WHERE room_id=? AND (disbanded_at IS NULL OR ?)""", (room_id, int(include_disbanded)), room_id) latest_seq = int(room["next_seq"]) - 1 - authority = { - "gateway_id": str(room["authority_gateway_id"]), "epoch": int(room["authority_epoch"]) - } + authority = {"gateway_id": str(room["authority_gateway_id"]), "epoch": int(room["authority_epoch"])} if since_seq > latest_seq: raise HostedRoomError("since_seq is ahead of the hosted room log") rows = conn.execute( @@ -1641,16 +1314,14 @@ def read_events( FROM candidates WHERE cumulative_bytes<=? ORDER BY seq ASC""", - (room_id, since_seq, limit, MAX_LOG_PAGE_BYTES), - ).fetchall() + (room_id, since_seq, limit, MAX_LOG_PAGE_BYTES)).fetchall() events = [_event_from_row(row) for row in rows] def build_page(page_events: list[dict[str, Any]]) -> dict[str, Any]: cursor = page_events[-1]["seq"] if page_events else since_seq return { "events": page_events, "cursor": cursor, "latest_seq": latest_seq, - "has_more": cursor < latest_seq, "authority": authority, - } + "has_more": cursor < latest_seq, "authority": authority} def page_bytes(page: dict[str, Any]) -> int: return len(json.dumps(page, ensure_ascii=False, separators=(",", ":")).encode("utf-8")) diff --git a/gateway/hosted_rooms_common.py b/gateway/hosted_rooms_common.py index a08de5b43a..923d72d558 100644 --- a/gateway/hosted_rooms_common.py +++ b/gateway/hosted_rooms_common.py @@ -1,11 +1,9 @@ """Shared leaf helpers for the gateway hosted-room modules. -Every hosted-room module validates identifiers, bounded integers, exact field -sets and canonical JSON with its own error class and its own error strings -(tests pin those strings). These helpers take the error class and message -templates as parameters so each caller keeps byte-identical failures while the -logic lives once. This module must stay a leaf: never import a hosted_room* -origin module from here (import cycle). +Each hosted-room module validates identifiers, bounded integers, exact field sets and +canonical JSON with its own error class and pinned error strings; these helpers take the +error class and message templates as parameters so failures stay byte-identical while the +logic lives once. Must stay a leaf: never import a hosted_room* module (import cycle). """ from __future__ import annotations @@ -13,7 +11,9 @@ from __future__ import annotations import json import re import sqlite3 +import time from contextlib import contextmanager +from functools import partial from pathlib import Path from typing import Any, Callable, Iterator, Mapping @@ -21,47 +21,33 @@ IDENTIFIER_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]*$") def identifier( - value: Any, - *, - label: str, - error: type[Exception], - max_chars: int = 128, - pattern: re.Pattern[str] = IDENTIFIER_RE, - invalid: str | None = None, -) -> str: - """Strip and validate a bounded identifier; ``invalid`` overrides the fail message.""" + value: Any, *, label: str, error: type[Exception], max_chars: int = 128, + pattern: re.Pattern[str] | None = IDENTIFIER_RE, invalid: str | None = None) -> str: + """Strip and validate a bounded string; ``pattern=None`` skips the shape check.""" if not isinstance(value, str): raise error(f"{label} must be a string") value = value.strip() - if not value or len(value) > max_chars or not pattern.fullmatch(value): + if not value or len(value) > max_chars or (pattern is not None and not pattern.fullmatch(value)): raise error(invalid or f"invalid {label}") return value -def positive_int(value: Any, *, error: type[Exception], message: str) -> int: - """Reject bools, non-ints and values below 1 (``message`` is the exact error text).""" - if isinstance(value, bool) or not isinstance(value, int) or value < 1: +def bounded_int(value: Any, *, error: type[Exception], message: str, low: int = 0, high: int | None = None) -> int: + """Reject bools, non-ints and values outside ``[low, high]`` (``message`` is the exact text).""" + if isinstance(value, bool) or not isinstance(value, int) or value < low or (high is not None and value > high): raise error(message) return value -def non_negative_int(value: Any, *, error: type[Exception], message: str) -> int: - if isinstance(value, bool) or not isinstance(value, int) or value < 0: - raise error(message) - return value +positive_int = partial(bounded_int, low=1) +non_negative_int = partial(bounded_int, low=0) def exact_fields( - value: Any, - *, - label: str, - required: frozenset[str] | set[str], - optional: frozenset[str] | set[str] = frozenset(), - error: type[Exception], - not_object: str | None = None, + value: Any, *, label: str, required: frozenset[str] | set[str], + optional: frozenset[str] | set[str] = frozenset(), error: type[Exception], not_object: str | None = None, missing_fmt: str = "{label} is missing fields: {fields}", - unknown_fmt: str = "{label} has unknown fields: {fields}", -) -> Mapping[str, Any]: + unknown_fmt: str = "{label} has unknown fields: {fields}") -> Mapping[str, Any]: """Require exactly ``required`` (+ any ``optional``) keys; formats name the offenders sorted.""" if not isinstance(value, Mapping): raise error(not_object or f"{label} must be an object") @@ -75,19 +61,28 @@ def exact_fields( return value -def canonical_json( - value: Any, - *, - error: type[Exception], - label: str, - max_bytes: int, - ensure_ascii: bool, -) -> str: - """Sorted, compact JSON bounded by ``max_bytes`` of UTF-8.""" +def text(value: Any, *, error: type[Exception], label: str, max_bytes: int, strip: bool = True) -> str: + """Non-blank string bounded by ``max_bytes`` of UTF-8; ``strip=False`` keeps and measures the raw text.""" + if not isinstance(value, str): + raise error(f"{label} must be a string") + if not value.strip(): + raise error(f"{label} must not be empty") + if strip: + value = value.strip() + if len(value.encode("utf-8")) > max_bytes: + raise error(f"{label} is too large") + return value + + +def compact_json(value: Any, *, ensure_ascii: bool = True) -> str: + """Sorted-key, separator-free JSON (the digest/storage canonical form).""" + return json.dumps(value, ensure_ascii=ensure_ascii, sort_keys=True, separators=(",", ":")) + + +def canonical_json(value: Any, *, error: type[Exception], label: str, max_bytes: int, ensure_ascii: bool) -> str: + """``compact_json`` bounded by ``max_bytes`` of UTF-8; unserializable input raises ``error``.""" try: - encoded = json.dumps( - value, ensure_ascii=ensure_ascii, sort_keys=True, separators=(",", ":") - ) + encoded = compact_json(value, ensure_ascii=ensure_ascii) except (TypeError, ValueError, RecursionError) as exc: raise error(f"{label} must be JSON-serializable") from exc if len(encoded.encode("utf-8")) > max_bytes: @@ -99,6 +94,11 @@ def utf8_len(*parts: str) -> int: return len("".join(parts).encode("utf-8")) +def clock(now: float | None) -> float: + """``now`` as a float, or the current wall clock when ``None``.""" + return time.time() if now is None else float(now) + + def open_sqlite(path: Path | str, *, timeout: float = 10) -> sqlite3.Connection: """Row-factory connection with foreign keys on; no journal or schema work.""" conn = sqlite3.connect(path, timeout=timeout) @@ -107,10 +107,46 @@ def open_sqlite(path: Path | str, *, timeout: float = 10) -> sqlite3.Connection: return conn +def connect( + db_path: Path | str, *, db_label: str, ready: Callable[[sqlite3.Connection], bool], + initialize: Callable[[sqlite3.Connection], None], lock_retries: int = 1) -> sqlite3.Connection: + """Open the shared root store: WAL, foreign keys, then ``initialize`` in one IMMEDIATE txn if not ``ready``. + + Multiple profile gateways share this database, so every draft-schema transition + is serialized in SQLite itself: a crash rolls back the whole DDL/data migration and + another process can safely retry it. Only the transient "database is locked" class + from the journal-mode pragma is retried (it may ignore the busy timeout while another + first opener initializes the DB, especially on Windows). + """ + from hermes_state import apply_wal_with_fallback + + path = Path(db_path) + path.parent.mkdir(parents=True, exist_ok=True) + conn = sqlite3.connect(path, timeout=10) + conn.row_factory = sqlite3.Row + try: + for attempt in range(lock_retries): + try: + apply_wal_with_fallback(conn, db_label=db_label) + break + except sqlite3.OperationalError as exc: + if str(exc).lower() != "database is locked" or attempt + 1 == lock_retries: + raise + time.sleep(0.01 * (2**attempt)) + conn.execute("PRAGMA foreign_keys=ON") + if not ready(conn): + conn.execute("BEGIN IMMEDIATE") + initialize(conn) + conn.commit() + except Exception: + conn.rollback() + conn.close() + raise + return conn + + def table_exists(conn: sqlite3.Connection, table: str) -> bool: - row = conn.execute( - "SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (table,) - ).fetchone() + row = conn.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (table,)).fetchone() return row is not None @@ -120,10 +156,7 @@ def table_columns(conn: sqlite3.Connection, table: str) -> frozenset[str]: @contextmanager def transaction( - connect: Callable[[Path | str], sqlite3.Connection], - db_path: Path | str, - *, - immediate: bool, + connect: Callable[[Path | str], sqlite3.Connection], db_path: Path | str, *, immediate: bool ) -> Iterator[sqlite3.Connection]: """Open via ``connect``, optionally ``BEGIN IMMEDIATE``, commit on success, always close.""" conn = connect(db_path) diff --git a/gateway/kanban_watchers.py b/gateway/kanban_watchers.py index 5929061f89..f38e6ee422 100644 --- a/gateway/kanban_watchers.py +++ b/gateway/kanban_watchers.py @@ -3,19 +3,27 @@ Background loops that subscribe to kanban boards, deliver notifications and artifacts, and drive the multi-agent dispatcher. They use only ``self`` state, so they live on a mixin ``GatewayRunner`` inherits. Per-tick work lives in -``kanban_watchers_notifier`` / ``kanban_watchers_dispatcher``. +``kanban_watchers_notifier`` / ``kanban_watchers_dispatcher``; shared plumbing +in ``kanban_watchers_common``. """ from __future__ import annotations import asyncio -import contextlib import os import time from pathlib import Path -from typing import Any, Callable, Optional +from typing import Any, Optional -from gateway.kanban_watchers_common import _to_thread_process_service, logger # noqa: F401 (tests import via origin) +from gateway.kanban_watchers_common import ( # noqa: F401 (tests import via origin) + _acquire_singleton_lock, + _kanban_dispatch_allowed, + _release_singleton_lock, + _resolve_auto_decompose_settings, + _gc_retention_days, + _to_thread_process_service, + logger, +) from gateway.kanban_watchers_notifier import ( # noqa: F401 (_wake_scope_id: tests import via origin) _KanbanNotification, _notifier_collect, @@ -27,100 +35,16 @@ from gateway.kanban_watchers_dispatcher import ( _resolve_dispatcher_settings, ) - -def _resolve_auto_decompose_settings( - load_config: Callable[[], Any], -) -> "tuple[bool, int]": - """Resolve the live (enabled, per_tick) auto-decompose settings. - - Read fresh every dispatcher tick so flipping ``kanban.auto_decompose: - false`` stops runaway fan-out on the next tick without a restart. Fails - safe: a config read error returns ``(False, 3)`` rather than re-enabling - a feature the user turned off. ``per_tick`` is clamped to ``>= 1``. - """ - try: - cfg = load_config() - except Exception: - return False, 3 - kcfg = cfg.get("kanban", {}) if isinstance(cfg, dict) else {} - enabled = bool(kcfg.get("auto_decompose", True)) - try: - per_tick = int(kcfg.get("auto_decompose_per_tick", 3) or 3) - except (TypeError, ValueError): - per_tick = 3 - return enabled, max(per_tick, 1) - - -def _kanban_dispatch_allowed() -> bool: - """Return False while the global emergency stop (`hermes pause`) is engaged. - - Checked every tick before spawning, so a pause applies on the next tick; - in-flight workers are never touched. Fails open if estop is unimportable. - """ - try: - from agent.estop import check_paused - except ImportError: - return True - return not check_paused("kanban", logger) - - -def _acquire_singleton_lock(lock_path) -> "tuple[Optional[object], str]": - """Take the exclusive, non-blocking advisory lock for the sole dispatcher. - - Only one gateway machine-wide may run the embedded dispatcher: concurrent - dispatchers double reclaim frequency and claim events, and with - ``wal_autocheckpoint=0`` concurrent manual checkpoints can corrupt index - pages. ``dispatch_in_gateway`` is the primary control; this is the backstop. - - Returns ``(handle, "held")`` (caller must release via - :func:`_release_singleton_lock`), ``(None, "contended")`` when another - process holds it (caller must NOT dispatch), or ``(None, "unavailable")`` - when locking cannot be performed (caller falls back to config control). - """ - try: - from gateway.status import _try_acquire_file_lock # deferred; same package - except ImportError: - return None, "unavailable" - try: - Path(lock_path).parent.mkdir(parents=True, exist_ok=True) - handle = open(str(lock_path), "a+", encoding="utf-8") - except OSError: - return None, "unavailable" - if not _try_acquire_file_lock(handle): - handle.close() - return None, "contended" - return handle, "held" - - -def _release_singleton_lock(handle) -> None: - """Release a lock acquired via :func:`_acquire_singleton_lock`.""" - if handle is None: - return - try: - from gateway.status import _release_file_lock - _release_file_lock(handle) - except Exception: - pass - with contextlib.suppress(Exception): - handle.close() - - -def _gc_retention_days() -> int: - """``kanban.done_sub_retention_days`` (default 30; 0 disables), re-read per sweep; fails safe to 30.""" - try: - from hermes_cli.config import load_config as _load_cfg - - _kanban_cfg = (_load_cfg() or {}).get("kanban") or {} - return int(_kanban_cfg.get("done_sub_retention_days", 30)) - except Exception: - return 30 +_IMAGE_EXTS = {".png", ".jpg", ".jpeg", ".gif", ".webp"} +_VIDEO_EXTS = {".mp4", ".mov", ".avi", ".mkv", ".webm", ".3gp"} +_GC_INTERVAL_SECONDS = 3600.0 +_HEALTH_WINDOW = 6 class GatewayKanbanWatchersMixin: """Kanban watcher / notifier / dispatcher loops for GatewayRunner.""" def _owns_kanban_dispatcher_lock(self) -> bool: - """Return whether this gateway currently owns the singleton lock.""" return getattr(self, "_kanban_dispatcher_lock_handle", None) is not None def _release_kanban_dispatcher_lock(self) -> None: @@ -141,15 +65,12 @@ class GatewayKanbanWatchersMixin: """Poll ``kanban_notify_subs`` and deliver terminal events to users. Per subscription, claims ``task_events`` newer than the stored cursor - (kinds in TERMINAL_KINDS), sends one message per event to - ``(platform, chat_id, thread_id)``, then advances the cursor. The - subscription is removed only when the task is ``archived``: ``done`` - is reversible (review/continuation), so the cursor — not unsubscribing - — is the dedup mechanism. Earlier unsub-on-terminal silently dropped - users when the dispatcher respawned a crashed task. - - All SQLite work runs in a thread; one tick's failure never stops the - next. Iterates every board on disk per tick. + (kinds in TERMINAL_KINDS), sends one message per event, then advances + the cursor. The subscription is removed only when the task is + ``archived``: ``done`` is reversible, so the cursor — not unsubscribing + — is the dedup mechanism (unsub-on-terminal dropped users when the + dispatcher respawned a crashed task). All SQLite work runs in a thread; + one tick's failure never stops the next. """ from gateway.config import Platform as _Platform try: @@ -158,23 +79,18 @@ class GatewayKanbanWatchersMixin: logger.warning("kanban notifier: kanban_db not importable; notifier disabled") return - sub_fail_counts: dict[tuple, int] = getattr( - self, "_kanban_sub_fail_counts", {} - ) + sub_fail_counts: dict[tuple, int] = getattr(self, "_kanban_sub_fail_counts", {}) self._kanban_sub_fail_counts = sub_fail_counts - notifier_profile = getattr(self, "_kanban_notifier_profile", None) - if not notifier_profile: - notifier_profile = self._active_profile_name() - self._kanban_notifier_profile = notifier_profile + notifier_profile = getattr(self, "_kanban_notifier_profile", None) or self._active_profile_name() + self._kanban_notifier_profile = notifier_profile # Initial delay so the gateway can finish wiring adapters. await asyncio.sleep(5) # Stale done-sub GC: subs survive ``done``, so boards that never # archive would accumulate rows scanned every tick. One DELETE per - # board, at startup and at most hourly. - _GC_INTERVAL_SECONDS = 3600.0 - _gc_next_at = 0.0 # 0 → sweep on the first tick after startup + # board, at startup (0 → first tick) and at most hourly. + _gc_next_at = 0.0 while self._running: try: @@ -186,9 +102,7 @@ class GatewayKanbanWatchersMixin: deliveries = await asyncio.to_thread( _notifier_collect, self, _kb, - notifier_profile=notifier_profile, - gc_due=_gc_due, - gc_retention_days=_retention, + notifier_profile=notifier_profile, gc_due=_gc_due, gc_retention_days=_retention, ) for d in deliveries: await _KanbanNotification( @@ -204,47 +118,23 @@ class GatewayKanbanWatchersMixin: conn = _kb.connect(board=board) try: getattr(_kb, op)( - conn, - task_id=sub["task_id"], - platform=sub["platform"], - chat_id=sub["chat_id"], - thread_id=sub.get("thread_id") or "", - **extra, + conn, task_id=sub["task_id"], platform=sub["platform"], chat_id=sub["chat_id"], + thread_id=sub.get("thread_id") or "", **extra, ) finally: conn.close() - def _kanban_advance( - self, sub: dict, cursor: int, board: Optional[str] = None, - ) -> None: - """Advance a subscription's cursor on the board that owns it.""" + def _kanban_advance(self, sub: dict, cursor: int, board: Optional[str] = None) -> None: self._kanban_sub_op(board, "advance_notify_cursor", sub, new_cursor=cursor) def _kanban_unsub(self, sub: dict, board: Optional[str] = None) -> None: self._kanban_sub_op(board, "remove_notify_sub", sub) - def _kanban_rewind( - self, - sub: dict, - claimed_cursor: int, - old_cursor: int, - board: Optional[str] = None, - ) -> None: + def _kanban_rewind(self, sub: dict, claimed_cursor: int, old_cursor: int, board: Optional[str] = None) -> None: """Undo a claimed notification cursor after send failure.""" - self._kanban_sub_op( - board, "rewind_notify_cursor", sub, - claimed_cursor=claimed_cursor, old_cursor=old_cursor, - ) + self._kanban_sub_op(board, "rewind_notify_cursor", sub, claimed_cursor=claimed_cursor, old_cursor=old_cursor) - async def _deliver_kanban_artifacts( - self, - *, - adapter, - chat_id: str, - metadata: dict, - event_payload: Optional[dict], - task, - ) -> None: + async def _deliver_kanban_artifacts(self, *, adapter, chat_id: str, metadata: dict, event_payload: Optional[dict], task) -> None: """Upload artifact files referenced by a completed kanban task. Sources, in priority order: ``event_payload['artifacts']``, @@ -252,36 +142,21 @@ class GatewayKanbanWatchersMixin: deduplicated, missing files are skipped (may be mentioned for reference only), and upload errors are logged, never raised. """ - candidates: list[str] = [] - seen: set[str] = set() - - def _add(path: str) -> None: - if not path: - return - expanded = os.path.expanduser(path) - if expanded in seen or not os.path.isfile(expanded): - return - seen.add(expanded) - candidates.append(expanded) - + raw_paths: list[str] = [] if isinstance(event_payload, dict): raw = event_payload.get("artifacts") if isinstance(raw, (list, tuple)): - for item in raw: - if isinstance(item, str): - _add(item) - + raw_paths += [item for item in raw if isinstance(item, str)] summary = event_payload.get("summary") if isinstance(summary, str) and summary: - paths, _ = adapter.extract_local_files(summary) - for p in paths: - _add(p) - + raw_paths += adapter.extract_local_files(summary)[0] if task is not None and getattr(task, "result", None): - paths, _ = adapter.extract_local_files(str(task.result)) - for p in paths: - _add(p) - + raw_paths += adapter.extract_local_files(str(task.result))[0] + candidates: list[str] = [] + for path in raw_paths: + expanded = os.path.expanduser(path) if path else "" + if expanded and expanded not in candidates and os.path.isfile(expanded): + candidates.append(expanded) if not candidates: return @@ -290,41 +165,73 @@ class GatewayKanbanWatchersMixin: if not candidates: return - _IMAGE_EXTS = {".png", ".jpg", ".jpeg", ".gif", ".webp"} - _VIDEO_EXTS = {".mp4", ".mov", ".avi", ".mkv", ".webm", ".3gp"} - from urllib.parse import quote as _quote # Images ride one send_multiple_images call (batch uploads on Signal/Slack). image_paths = [p for p in candidates if Path(p).suffix.lower() in _IMAGE_EXTS] other_paths = [p for p in candidates if Path(p).suffix.lower() not in _IMAGE_EXTS] - if image_paths: try: batch = [(f"file://{_quote(p)}", "") for p in image_paths] - await adapter.send_multiple_images( - chat_id=chat_id, images=batch, metadata=metadata, - ) + await adapter.send_multiple_images(chat_id=chat_id, images=batch, metadata=metadata) except Exception as exc: - logger.warning( - "kanban notifier: image batch upload failed: %s", exc, - ) - + logger.warning("kanban notifier: image batch upload failed: %s", exc) for path in other_paths: try: if Path(path).suffix.lower() in _VIDEO_EXTS: - await adapter.send_video( - chat_id=chat_id, video_path=path, metadata=metadata, - ) + await adapter.send_video(chat_id=chat_id, video_path=path, metadata=metadata) else: - await adapter.send_document( - chat_id=chat_id, file_path=path, metadata=metadata, - ) + await adapter.send_document(chat_id=chat_id, file_path=path, metadata=metadata) except Exception as exc: - logger.warning( - "kanban notifier: artifact upload (%s) failed: %s", - path, exc, - ) + logger.warning("kanban notifier: artifact upload (%s) failed: %s", path, exc) + + def _kanban_dispatcher_boot(self) -> Optional[tuple]: + """Resolve config, kanban_db and the singleton lock; None when the dispatcher must not run. + + Config is read once at boot (restart to apply), except the auto-decompose + toggle which is re-read every tick. The env var is an escape hatch to + disable without editing YAML. + """ + try: + from hermes_cli.config import load_config as _load_config + except Exception: + logger.warning("kanban dispatcher: config loader unavailable; disabled") + return None + env_override = os.environ.get("HERMES_KANBAN_DISPATCH_IN_GATEWAY", "").strip().lower() + if env_override in {"0", "false", "no", "off"}: + logger.info("kanban dispatcher: disabled via HERMES_KANBAN_DISPATCH_IN_GATEWAY env") + return None + try: + cfg = _load_config() + except Exception as exc: + logger.warning("kanban dispatcher: cannot load config (%s); disabled", exc) + return None + kanban_cfg = cfg.get("kanban", {}) if isinstance(cfg, dict) else {} + if not kanban_cfg.get("dispatch_in_gateway", True): + logger.info("kanban dispatcher: disabled via config kanban.dispatch_in_gateway=false") + return None + try: + from hermes_cli import kanban_db as _kb + except Exception: + logger.warning("kanban dispatcher: kanban_db not importable; dispatcher disabled") + return None + + # Single-dispatcher backstop (see _acquire_singleton_lock). The lock + # lives at the machine-global kanban root, so it serialises ALL gateways. + self._kanban_dispatcher_lock_handle = None + _lock_path = _kb.kanban_home() / "kanban" / ".dispatcher.lock" + _lock_handle, _lock_state = _acquire_singleton_lock(_lock_path) + if _lock_state == "contended": + logger.info("kanban dispatcher: another gateway already holds the dispatcher " + "lock (%s); this gateway will NOT dispatch.", _lock_path) + return None + if _lock_state == "held": + self._kanban_dispatcher_lock_handle = _lock_handle # hold for process lifetime + logger.info("kanban dispatcher: holding singleton dispatcher lock (%s)", _lock_path) + else: + logger.warning("kanban dispatcher: advisory lock unavailable at %s; proceeding " + "on config control alone.", _lock_path) + return _load_config, _kb, kanban_cfg async def _kanban_dispatcher_watcher(self) -> None: """Embedded kanban dispatcher — one tick every `dispatch_interval_seconds`. @@ -335,57 +242,10 @@ class GatewayKanbanWatchersMixin: failure never stops the next. Shutdown: ``self._running`` is checked between ticks and the in-flight ``to_thread`` returns on its own. """ - # Config is read once at boot (restart to apply), except the - # auto-decompose toggle which is re-read every tick. The env var is - # an escape hatch to disable without editing YAML. - try: - from hermes_cli.config import load_config as _load_config - except Exception: - logger.warning("kanban dispatcher: config loader unavailable; disabled") + boot = self._kanban_dispatcher_boot() + if boot is None: return - env_override = os.environ.get("HERMES_KANBAN_DISPATCH_IN_GATEWAY", "").strip().lower() - if env_override in {"0", "false", "no", "off"}: - logger.info("kanban dispatcher: disabled via HERMES_KANBAN_DISPATCH_IN_GATEWAY env") - return - - try: - cfg = _load_config() - except Exception as exc: - logger.warning("kanban dispatcher: cannot load config (%s); disabled", exc) - return - kanban_cfg = cfg.get("kanban", {}) if isinstance(cfg, dict) else {} - if not kanban_cfg.get("dispatch_in_gateway", True): - logger.info( - "kanban dispatcher: disabled via config kanban.dispatch_in_gateway=false" - ) - return - - try: - from hermes_cli import kanban_db as _kb - except Exception: - logger.warning("kanban dispatcher: kanban_db not importable; dispatcher disabled") - return - - # Single-dispatcher backstop (see _acquire_singleton_lock). The lock - # lives at the machine-global kanban root, so it serialises ALL gateways. - self._kanban_dispatcher_lock_handle = None - _lock_path = _kb.kanban_home() / "kanban" / ".dispatcher.lock" - _lock_handle, _lock_state = _acquire_singleton_lock(_lock_path) - if _lock_state == "contended": - logger.info( - "kanban dispatcher: another gateway already holds the dispatcher " - "lock (%s); this gateway will NOT dispatch.", _lock_path, - ) - return - if _lock_state == "held": - self._kanban_dispatcher_lock_handle = _lock_handle # hold for process lifetime - logger.info("kanban dispatcher: holding singleton dispatcher lock (%s)", _lock_path) - else: - logger.warning( - "kanban dispatcher: advisory lock unavailable at %s; proceeding " - "on config control alone.", _lock_path, - ) - + _load_config, _kb, kanban_cfg = boot settings = _resolve_dispatcher_settings(kanban_cfg, _kb) interval = settings.interval @@ -395,25 +255,18 @@ class GatewayKanbanWatchersMixin: # Health telemetry (mirrors `_cmd_daemon`): warn when the ready queue # is non-empty but spawns are 0 for N consecutive ticks — usually a # broken PATH, missing venv, or credential loss. - HEALTH_WINDOW = 6 bad_ticks = 0 last_warn_at = 0 dispatcher = _KanbanDispatcher(_kb, settings) - logger.info( - "kanban dispatcher: embedded in gateway (interval=%.1fs)", interval - ) + logger.info("kanban dispatcher: embedded in gateway (interval=%.1fs)", interval) while self._running: try: # Reap zombies before per-board work so a board DB failure # cannot block cleanup of unrelated workers. pids = await _to_thread_process_service(_kb.reap_worker_zombies) if pids: - logger.info( - "kanban dispatcher: reaped %d zombie worker(s), pids=%s", - len(pids), - pids, - ) + logger.info("kanban dispatcher: reaped %d zombie worker(s), pids=%s", len(pids), pids) except Exception: logger.exception("kanban dispatcher: zombie reaper failed") @@ -430,20 +283,18 @@ class GatewayKanbanWatchersMixin: await _to_thread_process_service(dispatcher.auto_decompose_tick, _ad_per_tick) results = await _to_thread_process_service(dispatcher.tick_once) any_spawned = _log_spawn_results(results) - # Health telemetry (aggregate across boards) ready_pending = await _to_thread_process_service(dispatcher.ready_nonempty) bad_ticks = bad_ticks + 1 if ready_pending and not any_spawned else 0 - if bad_ticks >= HEALTH_WINDOW: - now = int(time.time()) - if now - last_warn_at >= 300: - logger.warning( - "kanban dispatcher stuck: ready queue non-empty for " - "%d consecutive ticks but 0 workers spawned. Check " - "profile health (venv, PATH, credentials) and " - "`hermes kanban list --status ready`.", - bad_ticks, - ) - last_warn_at = now + now = int(time.time()) + if bad_ticks >= _HEALTH_WINDOW and now - last_warn_at >= 300: + logger.warning( + "kanban dispatcher stuck: ready queue non-empty for " + "%d consecutive ticks but 0 workers spawned. Check " + "profile health (venv, PATH, credentials) and " + "`hermes kanban list --status ready`.", + bad_ticks, + ) + last_warn_at = now except asyncio.CancelledError: logger.debug("kanban dispatcher: cancelled") self._release_kanban_dispatcher_lock() diff --git a/gateway/kanban_watchers_common.py b/gateway/kanban_watchers_common.py index 22833575ee..167e24d49b 100644 --- a/gateway/kanban_watchers_common.py +++ b/gateway/kanban_watchers_common.py @@ -1,11 +1,18 @@ -"""Plumbing shared by the kanban notifier and dispatcher loops.""" +"""Plumbing shared by the kanban notifier and dispatcher loops. + +Thread offload, board enumeration, live-config coercers and the singleton +dispatcher lock live here so the notifier, dispatcher and mixin modules read +them from one place. +""" from __future__ import annotations import asyncio +import contextlib import logging from contextvars import Context -from typing import Any, Callable +from pathlib import Path +from typing import Any, Callable, Optional # Keep the logger name run.py used so extracted log records are unchanged. logger = logging.getLogger("gateway.run") @@ -33,3 +40,105 @@ def _list_boards(kb: Any) -> list: return kb.list_boards(include_archived=False) except Exception: return [kb.read_board_metadata(kb.DEFAULT_BOARD)] + + +def _board_slugs(kb: Any) -> list: + return [b.get("slug") or kb.DEFAULT_BOARD for b in _list_boards(kb)] + + +def _positive_int_setting(kanban_cfg: dict, key: str) -> Optional[int]: + """Parse an optional ``kanban.`` int cap; None when unset or invalid (< 1 is invalid).""" + raw = kanban_cfg.get(key) + if raw is None: + return None + try: + value = int(raw) + except (TypeError, ValueError): + logger.warning("kanban dispatcher: invalid kanban.%s=%r; ignoring", key, raw) + return None + if value < 1: + logger.warning("kanban dispatcher: kanban.%s=%r is below 1; ignoring", key, raw) + return None + logger.info("kanban dispatcher: %s=%d", key, value) + return value + + +def _resolve_auto_decompose_settings(load_config: Callable[[], Any]) -> "tuple[bool, int]": + """Live (enabled, per_tick) auto-decompose settings, re-read every dispatcher tick. + + Fails safe: a config read error returns ``(False, 3)`` rather than + re-enabling a feature the user turned off. ``per_tick`` is clamped to ``>= 1``. + """ + try: + cfg = load_config() + except Exception: + return False, 3 + kcfg = cfg.get("kanban", {}) if isinstance(cfg, dict) else {} + try: + per_tick = int(kcfg.get("auto_decompose_per_tick", 3) or 3) + except (TypeError, ValueError): + per_tick = 3 + return bool(kcfg.get("auto_decompose", True)), max(per_tick, 1) + + +def _gc_retention_days() -> int: + """``kanban.done_sub_retention_days`` (default 30; 0 disables), re-read per sweep; fails safe to 30.""" + try: + from hermes_cli.config import load_config + + return int(((load_config() or {}).get("kanban") or {}).get("done_sub_retention_days", 30)) + except Exception: + return 30 + + +def _kanban_dispatch_allowed() -> bool: + """False while the global emergency stop (`hermes pause`) is engaged. + + Checked every tick before spawning, so a pause applies on the next tick; + in-flight workers are never touched. Fails open if estop is unimportable. + """ + try: + from agent.estop import check_paused + except ImportError: + return True + return not check_paused("kanban", logger) + + +def _acquire_singleton_lock(lock_path) -> "tuple[Optional[object], str]": + """Take the exclusive, non-blocking advisory lock for the sole dispatcher. + + Only one gateway machine-wide may run the embedded dispatcher: concurrent + dispatchers double reclaim frequency and claim events, and with + ``wal_autocheckpoint=0`` concurrent manual checkpoints can corrupt index + pages. ``dispatch_in_gateway`` is the primary control; this is the backstop. + + Returns ``(handle, "held")`` (release via :func:`_release_singleton_lock`), + ``(None, "contended")`` when another process holds it (caller must NOT + dispatch), or ``(None, "unavailable")`` when locking cannot be performed + (caller falls back to config control). + """ + try: + from gateway.status import _try_acquire_file_lock # deferred; same package + except ImportError: + return None, "unavailable" + try: + Path(lock_path).parent.mkdir(parents=True, exist_ok=True) + handle = open(str(lock_path), "a+", encoding="utf-8") + except OSError: + return None, "unavailable" + if not _try_acquire_file_lock(handle): + handle.close() + return None, "contended" + return handle, "held" + + +def _release_singleton_lock(handle) -> None: + """Release a lock acquired via :func:`_acquire_singleton_lock`.""" + if handle is None: + return + with contextlib.suppress(Exception): + from gateway.status import _release_file_lock + + _release_file_lock(handle) + with contextlib.suppress(Exception): + handle.close() diff --git a/gateway/kanban_watchers_dispatcher.py b/gateway/kanban_watchers_dispatcher.py index b768906245..28688b6068 100644 --- a/gateway/kanban_watchers_dispatcher.py +++ b/gateway/kanban_watchers_dispatcher.py @@ -11,10 +11,10 @@ import contextlib import os import sqlite3 import time -from dataclasses import dataclass +from dataclasses import asdict, dataclass from typing import Any, Optional -from gateway.kanban_watchers_common import _list_boards, logger +from gateway.kanban_watchers_common import _board_slugs, _positive_int_setting, logger _CORRUPT_DB_MARKERS = ("file is not a database", "database disk image is malformed") @@ -33,32 +33,13 @@ class _DispatcherSettings: max_in_progress_per_profile: Optional[int] -def _positive_int_setting(kanban_cfg: dict, key: str) -> Optional[int]: - """Parse an optional ``kanban.`` int cap; None when unset or invalid (< 1 is invalid).""" - raw = kanban_cfg.get(key) - if raw is None: - return None - try: - value = int(raw) - except (TypeError, ValueError): - logger.warning("kanban dispatcher: invalid kanban.%s=%r; ignoring", key, raw) - return None - if value < 1: - logger.warning("kanban dispatcher: kanban.%s=%r is below 1; ignoring", key, raw) - return None - logger.info("kanban dispatcher: %s=%d", key, value) - return value - - def _resolve_dispatcher_settings(kanban_cfg: dict, kb: Any) -> _DispatcherSettings: """Parse and log the dispatcher settings in their established order.""" try: interval = float(kanban_cfg.get("dispatch_interval_seconds", 60) or 60) except (ValueError, TypeError): - logger.warning( - "kanban dispatcher: invalid dispatch_interval_seconds=%r, using default 60", - kanban_cfg.get("dispatch_interval_seconds"), - ) + logger.warning("kanban dispatcher: invalid dispatch_interval_seconds=%r, using default 60", + kanban_cfg.get("dispatch_interval_seconds")) interval = 60.0 interval = max(interval, 1.0) # sanity floor — tighter than this is a footgun @@ -66,10 +47,9 @@ def _resolve_dispatcher_settings(kanban_cfg: dict, kb: Any) -> _DispatcherSettin if max_spawn is not None: logger.info("kanban dispatcher: max_spawn=%s", max_spawn) - # Cap simultaneously running tasks so slow workers don't pile up and - # time out. Explicit config wins; otherwise a memory-derived default - # (unbounded fan-out swap-thrashes small hosts), or None where total - # memory can't be read. + # Cap simultaneously running tasks so slow workers don't pile up and time + # out. Explicit config wins; otherwise a memory-derived default (unbounded + # fan-out swap-thrashes small hosts), or None where total memory can't be read. max_in_progress = _positive_int_setting(kanban_cfg, "max_in_progress") effective_max_in_progress = kb.resolve_max_in_progress(max_in_progress) if max_in_progress is None and effective_max_in_progress is not None: @@ -84,18 +64,12 @@ def _resolve_dispatcher_settings(kanban_cfg: dict, kb: Any) -> _DispatcherSettin try: failure_limit = int(raw_failure_limit) except (TypeError, ValueError): - logger.warning( - "kanban dispatcher: invalid kanban.failure_limit=%r; using default %d", - raw_failure_limit, - kb.DEFAULT_FAILURE_LIMIT, - ) + logger.warning("kanban dispatcher: invalid kanban.failure_limit=%r; using default %d", + raw_failure_limit, kb.DEFAULT_FAILURE_LIMIT) failure_limit = kb.DEFAULT_FAILURE_LIMIT if failure_limit < 1: - logger.warning( - "kanban dispatcher: kanban.failure_limit=%r is below 1; using default %d", - raw_failure_limit, - kb.DEFAULT_FAILURE_LIMIT, - ) + logger.warning("kanban dispatcher: kanban.failure_limit=%r is below 1; using default %d", + raw_failure_limit, kb.DEFAULT_FAILURE_LIMIT) failure_limit = kb.DEFAULT_FAILURE_LIMIT # 0 disables stale detection. @@ -103,22 +77,16 @@ def _resolve_dispatcher_settings(kanban_cfg: dict, kb: Any) -> _DispatcherSettin try: stale_timeout_seconds = int(raw_stale or 0) except (TypeError, ValueError): - logger.warning( - "kanban dispatcher: invalid kanban.dispatch_stale_timeout_seconds=%r; " - "disabling stale detection", - raw_stale, - ) + logger.warning("kanban dispatcher: invalid kanban.dispatch_stale_timeout_seconds=%r; " + "disabling stale detection", raw_stale) stale_timeout_seconds = 0 # Fallback profile for tasks created without an assignee (e.g. via the # dashboard). Empty (the schema default) keeps skipping them. default_assignee = (kanban_cfg.get("default_assignee") or "").strip() or None if default_assignee: - logger.info( - "kanban dispatcher: default_assignee=%r (unassigned ready tasks " - "will route to this profile)", - default_assignee, - ) + logger.info("kanban dispatcher: default_assignee=%r (unassigned ready tasks " + "will route to this profile)", default_assignee) return _DispatcherSettings( interval=interval, @@ -150,12 +118,10 @@ class _KanbanDispatcher: def __init__(self, kb: Any, settings: _DispatcherSettings) -> None: self.kb = kb self.settings = settings - self.disabled_corrupt_boards: dict[ - str, tuple[tuple[str, int | None, int | None], float] - ] = {} + self.disabled_corrupt_boards: dict[str, tuple[tuple[str, int | None, int | None], float]] = {} def _board_slugs(self) -> list: - return [b.get("slug") or self.kb.DEFAULT_BOARD for b in _list_boards(self.kb)] + return _board_slugs(self.kb) def board_db_fingerprint(self, slug: str) -> tuple[str, int | None, int | None]: path = self.kb.kanban_db_path(slug) @@ -173,10 +139,7 @@ class _KanbanDispatcher: corrupt_guard_error = getattr(self.kb, "KanbanDbCorruptError", None) if corrupt_guard_error is not None and isinstance(exc, corrupt_guard_error): return True - if not isinstance(exc, sqlite3.DatabaseError): - return False - msg = str(exc).lower() - return any(marker in msg for marker in _CORRUPT_DB_MARKERS) + return isinstance(exc, sqlite3.DatabaseError) and any(m in str(exc).lower() for m in _CORRUPT_DB_MARKERS) def _quarantine_lifted(self, slug: str, fingerprint: tuple) -> bool: """Return False while *slug* stays quarantined; lift (and log) otherwise.""" @@ -188,17 +151,10 @@ class _KanbanDispatcher: if disabled_fingerprint == fingerprint and age < self.CORRUPT_BOARD_RETRY_AFTER_SECONDS: return False if disabled_fingerprint == fingerprint: - logger.info( - "kanban dispatcher: board %s database fingerprint unchanged " - "after %.0fs quarantine; retrying dispatch", - slug, - age, - ) + logger.info("kanban dispatcher: board %s database fingerprint unchanged " + "after %.0fs quarantine; retrying dispatch", slug, age) else: - logger.info( - "kanban dispatcher: board %s database changed; retrying dispatch", - slug, - ) + logger.info("kanban dispatcher: board %s database changed; retrying dispatch", slug) self.disabled_corrupt_boards.pop(slug, None) return True @@ -212,22 +168,12 @@ class _KanbanDispatcher: fingerprint = self.board_db_fingerprint(slug) if not self._quarantine_lifted(slug, fingerprint): return None - s = self.settings + kwargs = {k: v for k, v in asdict(self.settings).items() if k != "interval"} try: # No explicit init_db(): connect() runs the migration once per # process (see the matching note in the notifier collector). conn = self.kb.connect(board=slug) - return self.kb.dispatch_once( - conn, - board=slug, - max_spawn=s.max_spawn, - max_in_progress=s.max_in_progress, - failure_limit=s.failure_limit, - stale_timeout_seconds=s.stale_timeout_seconds, - default_assignee=s.default_assignee, - max_in_progress_per_profile=s.max_in_progress_per_profile, - reconcile_orphans=s.reconcile_orphans, - ) + return self.kb.dispatch_once(conn, board=slug, **kwargs) except Exception as exc: if self.is_corrupt_board_db_error(exc): self.disabled_corrupt_boards[slug] = (fingerprint, time.monotonic()) @@ -237,8 +183,7 @@ class _KanbanDispatcher: "the file changes, the gateway restarts, or the " "quarantine timer expires. Move or restore the file, " "then run `hermes kanban init` if you need a fresh board.", - slug, - fingerprint[0], + slug, fingerprint[0], ) return None logger.exception("kanban dispatcher: tick failed on board %s", slug) @@ -253,14 +198,13 @@ class _KanbanDispatcher: return [(slug, self.tick_once_for_board(slug)) for slug in self._board_slugs()] def ready_nonempty(self) -> bool: - """Is there a ready+assigned+unclaimed task on ANY board that the - dispatcher would actually spawn for? + """Is there a ready+assigned+unclaimed task on ANY board the dispatcher would spawn for? - Control-plane lanes (e.g. ``orion-cc``) are pulled by terminals - via ``claim_task`` and never spawnable — a queue full of those is - "correctly idle", not "stuck". The review column is probed only - when review dispatch is on (same gate as the dispatcher): a task - waiting for a human reviewer is idle, not stuck. + Control-plane lanes (e.g. ``orion-cc``) are pulled by terminals via + ``claim_task`` and never spawnable — a queue full of those is + "correctly idle", not "stuck". The review column is probed only when + review dispatch is on (same gate as the dispatcher): a task waiting + for a human reviewer is idle, not stuck. """ kb = self.kb _review_probe = kb.review_dispatch_enabled() @@ -268,9 +212,7 @@ class _KanbanDispatcher: conn = None try: conn = kb.connect(board=slug) - if kb.has_spawnable_ready(conn): - return True - if _review_probe and kb.has_spawnable_review(conn): + if kb.has_spawnable_ready(conn) or (_review_probe and kb.has_spawnable_review(conn)): return True except Exception: continue @@ -281,17 +223,15 @@ class _KanbanDispatcher: return False def auto_decompose_tick(self, auto_decompose_per_tick: int) -> int: - """Auto-decompose up to N triage tasks across all boards into - ready workgraphs before dispatch fans out; the per-tick cap keeps - a bulk triage load from burst-spending the aux LLM. Returns the - number decomposed/specified. + """Auto-decompose up to N triage tasks across all boards into ready workgraphs. + + Runs before dispatch fans out; the per-tick cap keeps a bulk triage + load from burst-spending the aux LLM. Returns the number decomposed. """ try: from hermes_cli import kanban_decompose as _decomp except Exception as exc: # pragma: no cover - logger.warning( - "kanban auto-decompose: import failed (%s); skipping", exc, - ) + logger.warning("kanban auto-decompose: import failed (%s); skipping", exc) return 0 attempted = 0 successes = 0 @@ -306,10 +246,7 @@ class _KanbanDispatcher: try: triage_ids = _decomp.list_triage_ids() except Exception as exc: - logger.debug( - "kanban auto-decompose: list_triage_ids failed on board %s (%s)", - slug, exc, - ) + logger.debug("kanban auto-decompose: list_triage_ids failed on board %s (%s)", slug, exc) triage_ids = [] for tid in triage_ids: if attempted >= auto_decompose_per_tick: @@ -329,28 +266,16 @@ class _KanbanDispatcher: try: outcome = _decomp.decompose_task(tid, author="auto-decomposer") except Exception: - logger.exception( - "kanban auto-decompose: decompose_task crashed on %s", - tid, - ) + logger.exception("kanban auto-decompose: decompose_task crashed on %s", tid) return 0 if not outcome.ok: # Common no-op reasons (no aux client) must not spam logs every tick. - logger.debug( - "kanban auto-decompose [%s]: %s skipped: %s", - slug, tid, outcome.reason, - ) + logger.debug("kanban auto-decompose [%s]: %s skipped: %s", slug, tid, outcome.reason) return 0 if outcome.fanout and outcome.child_ids: - logger.info( - "kanban auto-decompose [%s]: %s → %d children", - slug, tid, len(outcome.child_ids), - ) + logger.info("kanban auto-decompose [%s]: %s → %d children", slug, tid, len(outcome.child_ids)) else: - logger.info( - "kanban auto-decompose [%s]: %s → single task (no fanout)", - slug, tid, - ) + logger.info("kanban auto-decompose [%s]: %s → single task (no fanout)", slug, tid) return 1 @@ -364,9 +289,7 @@ def _log_spawn_results(results: Optional[list]) -> bool: logger.info( "kanban dispatcher [%s]: spawned=%d reclaimed=%d " "crashed=%d timed_out=%d promoted=%d auto_blocked=%d", - slug, - len(res.spawned), - res.reclaimed, + slug, len(res.spawned), res.reclaimed, len(res.crashed) if hasattr(res.crashed, "__len__") else 0, len(res.timed_out) if hasattr(res.timed_out, "__len__") else 0, res.promoted, diff --git a/gateway/kanban_watchers_notifier.py b/gateway/kanban_watchers_notifier.py index 2abb00890e..128b7ec037 100644 --- a/gateway/kanban_watchers_notifier.py +++ b/gateway/kanban_watchers_notifier.py @@ -21,34 +21,21 @@ from gateway.kanban_watchers_common import _list_boards, _to_thread_process_serv TERMINAL_KINDS = ("completed", "blocked", "gave_up", "crashed", "timed_out", "status", "archived", "unblocked", "block_loop_detected", "review_requested", "changes_requested") # Kinds that hand a decision back to the origin, which must take a turn. # status/archived/unblocked are bookkeeping. -_WAKE_KINDS = ( - "completed", "gave_up", "crashed", "timed_out", - "blocked", "review_requested", "changes_requested", - "block_loop_detected", -) -# Consecutive send failures (adapter raised OR reported -# SendResult(success=False)) before a sub is dropped as a dead chat. -# 12 ≈ 60s at the 5s cadence: a transient API outage must not -# permanently unsubscribe a live review-gate channel. +_WAKE_KINDS = ("completed", "gave_up", "crashed", "timed_out", "blocked", "review_requested", "changes_requested", "block_loop_detected") +# Consecutive send failures (adapter raised OR reported SendResult(success=False)) +# before a sub is dropped as a dead chat. 12 ≈ 60s at the 5s cadence: a transient +# API outage must not permanently unsubscribe a live review-gate channel. MAX_SEND_FAILURES = 12 -_LOCAL_PATH_RE = re.compile( - r"(? str: """Return a mobile-friendly review reason safe for external delivery.""" from agent.redact import redact_sensitive_text - reason = redact_sensitive_text( - "" if value is None else str(value), - force=True, - redact_url_credentials=True, - ) - reason = _LOCAL_PATH_RE.sub("[local path]", reason) - reason = " ".join(reason.split()) + reason = redact_sensitive_text("" if value is None else str(value), force=True, redact_url_credentials=True) + reason = " ".join(_LOCAL_PATH_RE.sub("[local path]", reason).split()) if len(reason) > limit: reason = reason[: limit - 1].rstrip() + "…" return reason @@ -70,21 +57,15 @@ def _wake_scope_id(adapter: Any, sub: dict) -> Optional[str]: if value: return str(value) resolver = getattr(adapter, "scope_id_for_chat", None) - if callable(resolver): - try: - resolved = resolver(str(sub.get("chat_id") or "")) - except Exception as exc: - # An adapter-side lookup failure yields no scope, never an error. - logger.debug( - "kanban notifier: scope lookup failed for chat %s: %s", - sub.get("chat_id"), - exc, - exc_info=True, - ) - return None - if resolved: - return str(resolved) - return None + if not callable(resolver): + return None + try: + resolved = resolver(str(sub.get("chat_id") or "")) + except Exception as exc: + # An adapter-side lookup failure yields no scope, never an error. + logger.debug("kanban notifier: scope lookup failed for chat %s: %s", sub.get("chat_id"), exc, exc_info=True) + return None + return str(resolved) if resolved else None def _platform_names(mapping: Any) -> set[str]: @@ -92,260 +73,177 @@ def _platform_names(mapping: Any) -> set[str]: return {getattr(platform, "value", str(platform)).lower() for platform in mapping} -# --------------------------------------------------------------------------- -# Collection (runs in a worker thread) -# --------------------------------------------------------------------------- +# --- Collection (runs in a worker thread) --- -def _notifier_collect( - runner: Any, - kb: Any, - *, - notifier_profile: Optional[str], - gc_due: bool, - gc_retention_days: int, -) -> list[dict]: +class _Collector: + """One tick's claim state: which profiles/platforms this gateway serves and the GC gate.""" + + def __init__(self, runner: Any, kb: Any, *, notifier_profile: Optional[str], gc_due: bool, gc_retention_days: int) -> None: + self.kb = kb + self.notifier_profile = notifier_profile + self.gc_due = gc_due + self.gc_retention_days = gc_retention_days + self.deliveries: list[dict] = [] + self.include_unowned = runner._owns_kanban_dispatcher_lock() + self.profile_adapters = getattr(runner, "_profile_adapters", {}) + self.notifier_profiles = {notifier_profile} + self.notifier_profiles.update(str(p).strip() for p in self.profile_adapters if str(p).strip()) + # Include every platform any secondary profile has live. This is only a + # coarse pre-filter; the precise per-profile check (_authorization_adapter, + # no default fallback) runs at delivery and rewinds the claim if it + # resolves to None. An unclaimed event never retries, so dropping a + # secondary-profile sub here would lose it. + self.active_platforms = _platform_names(runner.adapters).union( + *(_platform_names(m) for m in self.profile_adapters.values())) + + def collect(self) -> list[dict]: + if not self.active_platforms: + logger.debug("kanban notifier: no connected adapters; skipping tick") + return self.deliveries + # Poll each resolved DB path once: several slugs can map to one DB when + # HERMES_KANBAN_DB pins the board path. + kb = self.kb + seen_db_paths: set[str] = set() + for board_meta in _list_boards(kb): + slug = board_meta.get("slug") or kb.DEFAULT_BOARD + db_path = board_meta.get("db_path") + try: + resolved_db_path = str(Path(db_path).expanduser().resolve()) if db_path else str(kb.kanban_db_path(slug).resolve()) + except Exception: + resolved_db_path = f"slug:{slug}" + if resolved_db_path in seen_db_paths: + logger.debug("kanban notifier: skipping duplicate board slug %s for DB %s", slug, resolved_db_path) + continue + seen_db_paths.add(resolved_db_path) + self.collect_board(slug) + return self.deliveries + + def _board_has_subs(self, slug: str) -> bool: + """Cheap read-only probe before the writable connect() (schema init, WAL + sidecars, checkpoints); a probe failure falls back to the writable open.""" + try: + count = self.kb.count_notify_subs( + board=slug, notifier_profiles=self.notifier_profiles, include_unowned=self.include_unowned) + except Exception as exc: + logger.debug("kanban notifier: read-only subscription probe failed " + "for board %s (%s); falling back to writable open", slug, exc) + return True + if count == 0: + logger.debug("kanban notifier: board %s has no subscriptions owned by %s; skipping open", + slug, sorted(self.notifier_profiles)) + return count != 0 + + def _gc_stale_subs(self, conn: Any, slug: str) -> None: + """Best-effort stale-sub sweep: a failed sweep never blocks delivery; the next hourly gate retries.""" + try: + _purged = self.kb.purge_stale_done_notify_subs(conn, max_age_days=self.gc_retention_days) + if _purged: + logger.info("kanban notifier: purged %d stale done/blocked-task subscription(s) on board %s (retention %dd)", + _purged, slug, self.gc_retention_days) + except Exception as _gc_exc: + logger.debug("kanban notifier: stale-sub GC failed for board %s: %s", slug, _gc_exc) + + def _claim_for_sub(self, conn: Any, slug: str, sub: dict) -> Optional[dict]: + """Claim one subscription's unseen events; None when skipped or nothing new.""" + owner_profile = sub.get("notifier_profile") or None + if owner_profile and owner_profile != self.notifier_profile and not self.profile_adapters.get(owner_profile): + logger.debug("kanban notifier: subscription for %s owned by profile %s; current profile %s has no adapter for it, skipping", + sub.get("task_id"), owner_profile, self.notifier_profile) + return None + platform = (sub.get("platform") or "").lower() + if platform not in self.active_platforms: + logger.debug("kanban notifier: subscription for %s on %s skipped; adapter not connected", + sub.get("task_id"), platform or "") + return None + old_cursor, cursor, events = self.kb.claim_unseen_events_for_sub( + conn, task_id=sub["task_id"], platform=sub["platform"], chat_id=sub["chat_id"], + thread_id=sub.get("thread_id") or "", kinds=TERMINAL_KINDS, + ) + if not events: + return None + task = self.kb.get_task(conn, sub["task_id"]) + logger.debug("kanban notifier: claimed %d event(s) for %s on board %s cursor %s→%s", + len(events), sub["task_id"], slug, old_cursor, cursor) + return {"sub": sub, "old_cursor": old_cursor, "cursor": cursor, "events": events, "task": task, "board": slug} + + def collect_board(self, slug: str) -> None: + """Claim events on one board, appending delivery dicts to ``deliveries``.""" + if not self._board_has_subs(slug): + return + kb = self.kb + try: + conn = kb.connect(board=slug) + except Exception as exc: + logger.debug("kanban notifier: cannot open board %s: %s", slug, exc) + return + try: + if self.gc_due: + self._gc_stale_subs(conn, slug) + # No explicit init_db(): connect() already runs the migration once per + # process, and init_db() would re-run it on a second connection racing + # the first. + subs = kb.list_notify_subs(conn, notifier_profiles=self.notifier_profiles, include_unowned=self.include_unowned) + if not subs: + logger.debug("kanban notifier: board %s has no subscriptions", slug) + for sub in subs: + try: + claimed = self._claim_for_sub(conn, slug, sub) + if claimed is not None: + self.deliveries.append(claimed) + except Exception as sub_exc: + # One bad subscription must not block the rest of the tick. + logger.warning("kanban notifier: subscription for %s on board %s failed: %s", + sub.get("task_id"), slug, sub_exc) + finally: + conn.close() + + +def _notifier_collect(runner: Any, kb: Any, *, notifier_profile: Optional[str], gc_due: bool, gc_retention_days: int) -> list[dict]: """Claim unseen terminal events for every owned subscription on every board. Each gateway polls only subscriptions owned by profiles whose adapters it hosts; legacy rows without a profile stamp are visible only to the process holding the singleton dispatcher lock. """ - deliveries: list[dict] = [] - include_unowned = runner._owns_kanban_dispatcher_lock() - profile_adapters = getattr(runner, "_profile_adapters", {}) - notifier_profiles = {notifier_profile} - notifier_profiles.update( - str(profile).strip() for profile in profile_adapters if str(profile).strip() - ) - active_platforms = _platform_names(runner.adapters) - # Include every platform any secondary profile has live. This is only a - # coarse pre-filter; the precise per-profile check (_authorization_adapter, - # no default fallback) runs at delivery and rewinds the claim if it - # resolves to None. An unclaimed event never retries, so dropping a - # secondary-profile sub here would lose it. - for _profile_adapter_map in profile_adapters.values(): - active_platforms.update(_platform_names(_profile_adapter_map)) - if not active_platforms: - logger.debug("kanban notifier: no connected adapters; skipping tick") - return deliveries - - # Poll each resolved DB path once: several slugs can map to one DB when - # HERMES_KANBAN_DB pins the board path. - seen_db_paths: set[str] = set() - for board_meta in _list_boards(kb): - slug = board_meta.get("slug") or kb.DEFAULT_BOARD - db_path = board_meta.get("db_path") - try: - resolved_db_path = str(Path(db_path).expanduser().resolve()) if db_path else str(kb.kanban_db_path(slug).resolve()) - except Exception: - resolved_db_path = f"slug:{slug}" - if resolved_db_path in seen_db_paths: - logger.debug( - "kanban notifier: skipping duplicate board slug %s for DB %s", - slug, resolved_db_path, - ) - continue - seen_db_paths.add(resolved_db_path) - _notifier_collect_board( - kb, slug, deliveries, - notifier_profile=notifier_profile, - notifier_profiles=notifier_profiles, - include_unowned=include_unowned, - profile_adapters=profile_adapters, - active_platforms=active_platforms, - gc_due=gc_due, - gc_retention_days=gc_retention_days, - ) - return deliveries + return _Collector( + runner, kb, notifier_profile=notifier_profile, gc_due=gc_due, gc_retention_days=gc_retention_days, + ).collect() -def _notifier_collect_board( - kb: Any, - slug: str, - deliveries: list[dict], - *, - notifier_profile: Optional[str], - notifier_profiles: set, - include_unowned: bool, - profile_adapters: dict, - active_platforms: set[str], - gc_due: bool, - gc_retention_days: int, -) -> None: - """Claim events on one board, appending delivery dicts to *deliveries*.""" - # Cheap read-only probe before the writable connect() (schema init, WAL - # sidecars, checkpoints) — a board with no subscriptions has nothing to notify. - try: - if kb.count_notify_subs( - board=slug, - notifier_profiles=notifier_profiles, - include_unowned=include_unowned, - ) == 0: - logger.debug( - "kanban notifier: board %s has no subscriptions owned by %s; skipping open", - slug, sorted(notifier_profiles), - ) - return - except Exception as exc: - logger.debug( - "kanban notifier: read-only subscription probe failed " - "for board %s (%s); falling back to writable open", - slug, exc, - ) - try: - conn = kb.connect(board=slug) - except Exception as exc: - logger.debug("kanban notifier: cannot open board %s: %s", slug, exc) - return - try: - if gc_due: - # Best-effort: a failed sweep never blocks delivery; the next - # hourly gate retries. - try: - _purged = kb.purge_stale_done_notify_subs(conn, max_age_days=gc_retention_days) - if _purged: - logger.info( - "kanban notifier: purged %d stale done/blocked-task subscription(s) on board %s (retention %dd)", - _purged, slug, gc_retention_days, - ) - except Exception as _gc_exc: - logger.debug( - "kanban notifier: stale-sub GC failed for board %s: %s", - slug, _gc_exc, - ) - # No explicit init_db(): connect() already runs the migration once per - # process, and init_db() would re-run it on a second connection racing - # the first. - subs = kb.list_notify_subs( - conn, - notifier_profiles=notifier_profiles, - include_unowned=include_unowned, - ) - if not subs: - logger.debug("kanban notifier: board %s has no subscriptions", slug) - for sub in subs: - try: - owner_profile = sub.get("notifier_profile") or None - if owner_profile and owner_profile != notifier_profile and not profile_adapters.get(owner_profile): - logger.debug( - "kanban notifier: subscription for %s owned by profile %s; current profile %s has no adapter for it, skipping", - sub.get("task_id"), owner_profile, notifier_profile, - ) - continue - platform = (sub.get("platform") or "").lower() - if platform not in active_platforms: - logger.debug( - "kanban notifier: subscription for %s on %s skipped; adapter not connected", - sub.get("task_id"), platform or "", - ) - continue - old_cursor, cursor, events = kb.claim_unseen_events_for_sub( - conn, - task_id=sub["task_id"], - platform=sub["platform"], - chat_id=sub["chat_id"], - thread_id=sub.get("thread_id") or "", - kinds=TERMINAL_KINDS, - ) - if not events: - continue - task = kb.get_task(conn, sub["task_id"]) - logger.debug( - "kanban notifier: claimed %d event(s) for %s on board %s cursor %s→%s", - len(events), sub["task_id"], slug, old_cursor, cursor, - ) - deliveries.append({ - "sub": sub, - "old_cursor": old_cursor, - "cursor": cursor, - "events": events, - "task": task, - "board": slug, - }) - except Exception as sub_exc: - # One bad subscription must not block the rest of the tick. - logger.warning( - "kanban notifier: subscription for %s on board %s failed: %s", - sub.get("task_id"), slug, sub_exc, - ) - finally: - conn.close() - - -# --------------------------------------------------------------------------- -# Per-event message formatting: kind -> (msg, wake_handoff, wake_review_detail) -# --------------------------------------------------------------------------- -# ``None`` for handoff / review_detail leaves the accumulated wake value -# untouched. ``_payload(ev, key)`` is the shared "payload present and truthy" read. +# --- Per-event message formatting: kind -> (msg, wake_handoff, wake_review_detail) --- +# ``None`` for handoff / review_detail leaves the accumulated wake value untouched. def _payload(ev: Any, key: str) -> Any: + """Shared "payload present and truthy" read.""" return ev.payload.get(key) if ev.payload and ev.payload.get(key) else None +def _clip(ev: Any, key: str, fmt: str, limit: int) -> str: + """``fmt`` applied to the truncated payload value, or ``""`` when absent.""" + value = _payload(ev, key) + return fmt.format(str(value)[:limit]) if value else "" + + +_NL = "\n{}" + + def _first_line(text: str, limit: int) -> str: lines = text.strip().splitlines() return lines[0][:limit] if lines else text[:limit] def _fmt_completed(ev, n) -> tuple: - # Prefer the run summary from the event payload; fall back to - # task.result for legacy rows. - handoff = "" + # Prefer the run summary from the event payload; fall back to task.result for legacy rows. wake_handoff = None payload_summary = _payload(ev, "summary") if payload_summary: wake_handoff = _first_line(str(payload_summary), 200) - handoff = f"\n{wake_handoff}" elif n.task and n.task.result: wake_handoff = _first_line(n.task.result, 160) - handoff = f"\n{wake_handoff}" - msg = ( - f"✔ {n.board_tag}{n.tag}Kanban {n.task_id} done" - f" — {n.title}{handoff}" - ) - return msg, wake_handoff, None - - -def _fmt_blocked(ev, n) -> tuple: - reason = _payload(ev, "reason") - reason = f": {str(reason)[:160]}" if reason else "" - return f"⏸ {n.board_tag}{n.tag}Kanban {n.task_id} blocked{reason}", None, None - - -def _fmt_gave_up(ev, n) -> tuple: - err = _payload(ev, "error") - err = f"\n{str(err)[:200]}" if err else "" - msg = ( - f"✖ {n.board_tag}{n.tag}Kanban {n.task_id} gave up " - f"after repeated spawn failures{err}" - ) - return msg, None, None - - -def _fmt_crashed(ev, n) -> tuple: - msg = ( - f"✖ {n.board_tag}{n.tag}Kanban {n.task_id} worker crashed " - f"(pid gone); dispatcher will retry" - ) - return msg, None, None - - -def _fmt_timed_out(ev, n) -> tuple: - limit = _payload(ev, "limit_seconds") - limit = int(limit) if limit else 0 - msg = ( - f"⏱ {n.board_tag}{n.tag}Kanban {n.task_id} timed out " - f"(max_runtime={limit}s); will retry" - ) - return msg, None, None - - -def _fmt_status(ev, n) -> tuple: - new_status = _payload(ev, "status") - new_status = str(new_status) if new_status else "" - return f"🔄 {n.board_tag}{n.tag}Kanban {n.task_id} → {new_status}", None, None + handoff = f"\n{wake_handoff}" if wake_handoff is not None else "" + return f"✔ {n.head} done — {n.title}{handoff}", wake_handoff, None def _fmt_review_requested(ev, n) -> tuple: @@ -358,11 +256,7 @@ def _fmt_review_requested(ev, n) -> tuple: summary = str(summary) handoff = f"\n{summary[:200]}" wake_handoff = _first_line(summary, 200) - msg = ( - f"👀 {n.board_tag}{n.tag}Kanban {n.task_id} ready for review" - f" — {n.title}{handoff}" - ) - return msg, wake_handoff, None + return f"👀 {n.head} ready for review — {n.title}{handoff}", wake_handoff, None def _fmt_changes_requested(ev, n) -> tuple: @@ -371,51 +265,40 @@ def _fmt_changes_requested(ev, n) -> tuple: reviewer = _safe_review_reason(payload.get("reviewer"), 48) implementer = _safe_review_reason(payload.get("implementer"), 48) reason_text = reason or "reviewer feedback requires changes" - provenance = "" - if reviewer: - provenance += f" — reviewer @{reviewer}" + provenance = f" — reviewer @{reviewer}" if reviewer else "" if implementer: provenance += f" → implementer @{implementer}" - msg = ( - f"🛑 {n.board_tag}Kanban {n.task_id} review requested " - f"changes/BLOCK: {reason_text}{provenance}" - ) + msg = f"🛑 {n.board_tag}Kanban {n.task_id} review requested changes/BLOCK: {reason_text}{provenance}" return msg, None, reason_text -def _fmt_block_loop_detected(ev, n) -> tuple: - # Re-blocked for the same cause past the limit and routed to `triage` - # for a human. It emits no blocked/status event, so ping loudly here. - reason = _payload(ev, "reason") - reason = f": {str(reason)[:160]}" if reason else "" - recurrences = ev.payload.get("recurrences") if ev.payload else None - rc = f" (blocked {recurrences}x for the same cause)" if recurrences else "" - msg = ( - f"🛑 {n.board_tag}{n.tag}Kanban {n.task_id} routed to TRIAGE" - f" — needs a human decision{rc}{reason}" - ) - return msg, None, None - - # archived / unblocked are claimed (so the cursor advances past them) but # intentionally silent (no formatter), and excluded from _WAKE_KINDS so they # never wake the creator. _EVENT_FORMATTERS: dict[str, Callable[[Any, "_KanbanNotification"], tuple]] = { "completed": _fmt_completed, - "blocked": _fmt_blocked, - "gave_up": _fmt_gave_up, - "crashed": _fmt_crashed, - "timed_out": _fmt_timed_out, - "status": _fmt_status, + "blocked": lambda ev, n: (f"⏸ {n.head} blocked{_clip(ev, 'reason', ': {}', 160)}", None, None), + "gave_up": lambda ev, n: ( + f"✖ {n.head} gave up after repeated spawn failures{_clip(ev, 'error', _NL, 200)}", None, None, + ), + "crashed": lambda ev, n: (f"✖ {n.head} worker crashed (pid gone); dispatcher will retry", None, None), + "timed_out": lambda ev, n: ( + f"⏱ {n.head} timed out (max_runtime={int(_payload(ev, 'limit_seconds') or 0)}s); will retry", None, None, + ), + "status": lambda ev, n: (f"🔄 {n.head} → {_payload(ev, 'status') or ''}", None, None), "review_requested": _fmt_review_requested, "changes_requested": _fmt_changes_requested, - "block_loop_detected": _fmt_block_loop_detected, + # Re-blocked for the same cause past the limit and routed to `triage` for a + # human. It emits no blocked/status event, so ping loudly here. + "block_loop_detected": lambda ev, n: ( + f"🛑 {n.head} routed to TRIAGE — needs a human decision" + f"{_clip(ev, 'recurrences', ' (blocked {}x for the same cause)', 200)}{_clip(ev, 'reason', ': {}', 160)}", + None, None, + ), } -# --------------------------------------------------------------------------- -# Delivery of one claimed batch (one subscription, N events) -# --------------------------------------------------------------------------- +# --- Delivery of one claimed batch (one subscription, N events) --- class _KanbanNotification: @@ -443,8 +326,8 @@ class _KanbanNotification: self.title = (task.title if task else sub["task_id"])[:120] self.board_tag = f"[{self.board_slug}] " if self.board_slug else "" # Attribute the ping to the worker that did the work. - who = task.assignee if task and task.assignee else None - self.tag = f"@{who} " if who else "" + tag = f"@{task.assignee} " if task and task.assignee else "" + self.head = f"{self.board_tag}{tag}Kanban {self.task_id}" # The wake self-post path needs the key even when every event was skipped. self.sub_key = (sub["task_id"], sub["platform"], sub["chat_id"], sub.get("thread_id") or "") mode = sub.get("delivery_mode") or "notify" @@ -452,14 +335,11 @@ class _KanbanNotification: self.send_passive = mode != "wake" # Worker handoff carried into the synthetic wake turn so the woken # creator doesn't re-decompose work already on the board. - self.wake_handoff = "" - self.wake_review_detail = "" + self.wake_handoff = self.wake_review_detail = self.session_key = self.synth = "" self.plat: Any = None self.adapter: Any = None self.is_push_adapter = True self.wake_kinds: set = set() - self.session_key = "" - self.synth = "" # -- cursor / subscription ops (blocking, run in a fresh-context thread) -- @@ -490,11 +370,8 @@ class _KanbanNotification: await self.rewind() async def _wake_failed(self, fmt: str, exc: Exception) -> None: - await self.delivery_failed( - fmt, (self.task_id,), - "kanban notifier: dropping subscription %s on %s after %d consecutive wake failures", - exc, True, - ) + drop_fmt = "kanban notifier: dropping subscription %s on %s after %d consecutive wake failures" + await self.delivery_failed(fmt, (self.task_id,), drop_fmt, exc, True) # -- formatting -- @@ -513,11 +390,7 @@ class _KanbanNotification: def build_wake_text(self) -> None: """Set ``wake_kinds`` / ``session_key`` / ``synth`` for the wake paths.""" task, sub = self.task, self.sub - self.wake_kinds = ( - {ev.kind for ev in self.d["events"] if ev.kind in _WAKE_KINDS} - if self.wake_agent - else set() - ) + self.wake_kinds = {ev.kind for ev in self.d["events"] if ev.kind in _WAKE_KINDS} if self.wake_agent else set() if not self.wake_kinds: return if self.is_push_adapter: @@ -532,11 +405,8 @@ class _KanbanNotification: _status = t("gateway.kanban.wake.status_joiner").join(_parts) or t("gateway.kanban.wake.status_default") synth = t( "gateway.kanban.wake.message", - task_id=sub["task_id"], - status=_status, - title=self.title, - assignee=task.assignee if task else "", - board=self.board_slug, + task_id=sub["task_id"], status=_status, title=self.title, + assignee=task.assignee if task else "", board=self.board_slug, ) # Label as an automatic notification and carry the handoff so the # creator inspects the board instead of re-decomposing. @@ -547,36 +417,32 @@ class _KanbanNotification: self.synth = synth + "\n\n" + t("gateway.kanban.wake.guidance") def _log_woke(self) -> None: - logger.info( - "kanban notifier: woke agent for %s on %s/%s profile=%s events=%s", - self.task_id, self.platform_str, self.sub["chat_id"], self.sub_profile or "default", self.wake_kinds, - ) + logger.info("kanban notifier: woke agent for %s on %s/%s profile=%s events=%s", + self.task_id, self.platform_str, self.sub["chat_id"], self.sub_profile or "default", self.wake_kinds) - async def push_wake(self) -> None: - """Wake the creator session behind a push adapter; raises on failure.""" - from gateway.session import SessionSource + async def wake(self) -> None: + """Wake the creator session (raises on failure): push adapters get a full SessionSource, non-push a raw self-post.""" from gateway.wake import deliver_wake sub = self.sub - # Rebuild the creator's real session scope from the persisted - # chat_type: build_session_key() keys DMs differently from - # group/thread, so a hardcoded "group" mis-routed DM/thread creators - # into a fresh session. Legacy rows may carry chat_type in - # delivery_metadata; last resort is "group". A mismatch only degrades - # to a fresh session. + if not self.is_push_adapter: + await deliver_wake(self.adapter, text=self.synth, session_id=self.session_key) + self._log_woke() + return + from gateway.session import SessionSource + # Rebuild the creator's real session scope from the persisted chat_type: + # build_session_key() keys DMs differently from group/thread, so a + # hardcoded "group" mis-routed DM/thread creators into a fresh session. + # Legacy rows may carry chat_type in delivery_metadata; last resort is + # "group". A mismatch only degrades to a fresh session. _chat_type = str(sub.get("chat_type") or "").strip() if not _chat_type: _delivery_meta = sub.get("delivery_metadata") if isinstance(_delivery_meta, dict): _chat_type = str(_delivery_meta.get("chat_type") or "").strip() _source = SessionSource( - platform=self.plat, - chat_id=sub["chat_id"], - chat_type=_chat_type or "group", - thread_id=sub.get("thread_id") or None, - user_id=sub.get("user_id"), - user_id_alt=sub.get("user_id_alt"), - profile=self.sub_profile or None, - scope_id=_wake_scope_id(self.adapter, sub), + platform=self.plat, chat_id=sub["chat_id"], chat_type=_chat_type or "group", + thread_id=sub.get("thread_id") or None, user_id=sub.get("user_id"), user_id_alt=sub.get("user_id_alt"), + profile=self.sub_profile or None, scope_id=_wake_scope_id(self.adapter, sub), ) await deliver_wake(self.adapter, text=self.synth, session_id=self.session_key, source=_source) self._log_woke() @@ -593,30 +459,19 @@ class _KanbanNotification: # (else the event is lost); None / non-SendResult keeps the # "no exception == delivered" contract. if getattr(_send_res, "success", True) is False: - raise RuntimeError( - "adapter send() reported failure: " - f"{getattr(_send_res, 'error', None) or 'unknown error'}" - ) - logger.debug( - "kanban notifier: delivered %s event for %s to %s/%s on board %s", - ev.kind, self.task_id, self.platform_str, sub["chat_id"], self.board_slug, - ) + raise RuntimeError(f"adapter send() reported failure: {getattr(_send_res, 'error', None) or 'unknown error'}") + logger.debug("kanban notifier: delivered %s event for %s to %s/%s on board %s", + ev.kind, self.task_id, self.platform_str, sub["chat_id"], self.board_slug) # Upload artifact paths from the completion payload / legacy result as # native files. Only on ``completed`` so retries never spam attachments. if ev.kind == "completed": try: await self.runner._deliver_kanban_artifacts( - adapter=adapter, - chat_id=sub["chat_id"], - metadata=metadata, - event_payload=getattr(ev, "payload", None), - task=self.task, + adapter=adapter, chat_id=sub["chat_id"], metadata=metadata, + event_payload=getattr(ev, "payload", None), task=self.task, ) except Exception as art_exc: - logger.debug( - "kanban notifier: artifact delivery for %s failed: %s", - self.task_id, art_exc, - ) + logger.debug("kanban notifier: artifact delivery for %s failed: %s", self.task_id, art_exc) async def _send_pings(self) -> bool: """Send every text ping; False when a send failed (claim already rewound/dropped).""" @@ -630,10 +485,8 @@ class _KanbanNotification: # IS the delivery and resolves the failure counter. if not self.is_push_adapter and self.wake_agent: logger.debug( - "kanban notifier: adapter %s has no push " - "channel; skipping text ping for %s, relying " - "on wake self-post instead", - self.platform_str, self.task_id, + "kanban notifier: adapter %s has no push channel; skipping text ping for %s, relying " + "on wake self-post instead", self.platform_str, self.task_id, ) continue if not self.send_passive: @@ -644,10 +497,8 @@ class _KanbanNotification: self.clear_failures() except Exception as exc: await self.delivery_failed( - "kanban notifier: send failed for %s on %s (attempt %d/%d): %s", - (self.task_id, self.platform_str), - "kanban notifier: dropping subscription %s on %s after %d consecutive send failures", - exc, False, + "kanban notifier: send failed for %s on %s (attempt %d/%d): %s", (self.task_id, self.platform_str), + "kanban notifier: dropping subscription %s on %s after %d consecutive send failures", exc, False, ) return False return True @@ -664,10 +515,8 @@ class _KanbanNotification: # default) has no adapter. adapter = self.runner._authorization_adapter(self.plat, self.sub_profile or None) if adapter is None: - logger.debug( - "kanban notifier: adapter %s disconnected before delivery for %s; rewinding claim", - self.platform_str, self.task_id, - ) + logger.debug("kanban notifier: adapter %s disconnected before delivery for %s; rewinding claim", + self.platform_str, self.task_id) await self.rewind() return self.adapter = adapter @@ -677,29 +526,21 @@ class _KanbanNotification: if not await self._send_pings(): return # All text pings delivered (or skipped for non-push / wake-only). - task_terminal = self.task and self.task.status == "archived" self.build_wake_text() wake_kinds, is_push = self.wake_kinds, self.is_push_adapter - if not is_push and wake_kinds and self.session_key: - # Self-post IS the delivery: must succeed BEFORE the cursor advances. - from gateway.wake import deliver_wake + # Non-push self-post, or wake-only push sub: the wake IS the delivery + # and must succeed BEFORE the cursor advances. + if wake_kinds and (not self.send_passive if is_push else bool(self.session_key)): try: - await deliver_wake(adapter, text=self.synth, session_id=self.session_key) - self._log_woke() + await self.wake() self.clear_failures() except Exception as _wk_err: - await self._wake_failed("kanban notifier: wake self-post failed for %s (attempt %d/%d): %s", _wk_err) - return - - if is_push and not self.send_passive and wake_kinds: - # Wake-only push sub: the wake is the sole delivery and must - # succeed BEFORE the cursor advances. - try: - await self.push_wake() - self.clear_failures() - except Exception as _wk_err: - await self._wake_failed("kanban notifier: wake-only delivery failed for %s (attempt %d/%d): %s", _wk_err) + await self._wake_failed( + "kanban notifier: wake-only delivery failed for %s (attempt %d/%d): %s" if is_push + else "kanban notifier: wake self-post failed for %s (attempt %d/%d): %s", + _wk_err, + ) return # Delivery complete: advance the cursor (the dedup mechanism). @@ -711,12 +552,9 @@ class _KanbanNotification: # advanced; the wake stays best-effort, but log at WARNING so a # persistently failing wake is visible. try: - await self.push_wake() + await self.wake() except Exception as _wk_err: - logger.warning( - "kanban notifier: wakeup injection failed for %s: %s", - self.task_id, _wk_err, exc_info=True, - ) + logger.warning("kanban notifier: wakeup injection failed for %s: %s", self.task_id, _wk_err, exc_info=True) # Unsubscribe only on archive; ``done`` is reversible. - if task_terminal: + if self.task and self.task.status == "archived": await self.unsub() diff --git a/gateway/lifecycle_ledger.py b/gateway/lifecycle_ledger.py index 3985013923..a12f6c5aae 100644 --- a/gateway/lifecycle_ledger.py +++ b/gateway/lifecycle_ledger.py @@ -1,24 +1,12 @@ """Gateway lifecycle ledger — durable termination-reason evidence. -Graceful shutdowns already leave forensics (:mod:`gateway.shutdown_forensics`, -``gateway-exit-diag.log``); an **unclean death** (SIGKILL, kernel OOM kill, VM -death) runs no handler, so the next boot would otherwise have no record of it. - -A tiny state machine persisted to ``/state/gateway.lifecycle.json`` -closes that gap: - -* :func:`record_startup` reads the previous life's sentinel. ``phase == - "running"`` means no exit path ran → unclean death. The finding (with the - last heartbeat's memory sample) is appended to ``gateway-exit-diag.log`` as a - ``gateway.previous_unclean_exit`` record and logged at WARNING, then the - sentinel is rewritten as ``phase=running`` for the new life. -* :func:`mark_exited` rewrites the sentinel as ``phase=exited`` on every clean - exit path (``_exit_after_graceful_shutdown`` and the watchdog ``os._exit`` sites). -* :func:`sample_memory` is the cheap /proc snapshot the 30s loop heartbeat - embeds, so OOM crash cycles are classifiable from the volume alone. - -Everything here is best-effort: a forensics failure must never affect the -gateway lifecycle it is observing. +Graceful shutdowns leave forensics (``gateway-exit-diag.log``); an unclean +death (SIGKILL, kernel OOM, VM death) runs no handler. A sentinel at +``/state/gateway.lifecycle.json`` closes the gap: +:func:`record_startup` finds ``phase == "running"`` from the previous life → +unclean death, appended to the exit-diag log as ``gateway.previous_unclean_exit`` +and logged at WARNING; :func:`mark_exited` rewrites ``phase=exited`` on every +clean exit path. Best-effort: forensics must never affect the lifecycle. """ from __future__ import annotations @@ -28,74 +16,61 @@ import logging import os import sqlite3 import time +from contextlib import closing from datetime import datetime, timezone from pathlib import Path from typing import Any, Dict, Optional logger = logging.getLogger(__name__) -_LIFECYCLE_RELATIVE = ("state", "gateway.lifecycle.json") -_EXIT_DIAG_RELATIVE = ("logs", "gateway-exit-diag.log") -_STATE_DB_RELATIVE = ("state.db",) - -# Heuristic OOM-suspicion thresholds on the last heartbeat's memory sample. -# Deliberately conservative: only a hint; classification stays with the reader. -_LOW_MEM_AVAILABLE_KIB = 64 * 1024 # < 64 MiB available -_LOW_MEM_AVAILABLE_FRACTION = 0.05 # < 5% of MemTotal available - def _process_hermes_home() -> Path: """HERMES_HOME for process-level identity files (ignore task overrides).""" - val = os.environ.get("HERMES_HOME", "").strip() - if val: - return Path(val) from hermes_constants import get_hermes_home - return get_hermes_home() + val = os.environ.get("HERMES_HOME", "").strip() + return Path(val) if val else get_hermes_home() -def _home_path(home: Optional[Path], relative: tuple) -> Path: - base = home if home is not None else _process_hermes_home() - return base.joinpath(*relative) +def _home_path(home: Optional[Path], *relative: str) -> Path: + return (_process_hermes_home() if home is None else home).joinpath(*relative) def get_lifecycle_sentinel_path(home: Optional[Path] = None) -> Path: """Return ``/state/gateway.lifecycle.json``.""" - return _home_path(home, _LIFECYCLE_RELATIVE) + return _home_path(home, "state", "gateway.lifecycle.json") -def sample_memory() -> Dict[str, Any]: - """Cheap memory snapshot: own RSS + system availability + swap. +def _now_iso() -> str: + return datetime.now(timezone.utc).isoformat() - Pure ``/proc`` reads, Linux-only (returns ``{}`` elsewhere), never - raises. Values in KiB to match the kernel's units. - """ - sample: Dict[str, Any] = {} + +def _proc_fields(path: str, wanted: Dict[str, str]) -> Dict[str, int]: + """``{dst: int}`` for each ``src: dst`` key found in a ``Key: value`` /proc file.""" + found: Dict[str, int] = {} try: - with open("/proc/self/status", encoding="utf-8") as fh: - for line in fh: - if line.startswith("VmRSS:"): - sample["rss_kib"] = int(line.split()[1]) - break - except (OSError, ValueError, IndexError): - pass - try: - meminfo: Dict[str, int] = {} - wanted = {"MemTotal", "MemAvailable", "SwapTotal", "SwapFree"} - with open("/proc/meminfo", encoding="utf-8") as fh: + with open(path, encoding="utf-8") as fh: for line in fh: key = line.split(":", 1)[0] if key in wanted: - meminfo[key] = int(line.split()[1]) - if len(meminfo) == len(wanted): + found[wanted[key]] = int(line.split()[1]) + if len(found) == len(wanted): break - for src, dst in (("MemTotal", "mem_total_kib"), ("MemAvailable", "mem_available_kib")): - if src in meminfo: - sample[dst] = meminfo[src] - if "SwapTotal" in meminfo and "SwapFree" in meminfo: - sample["swap_used_kib"] = meminfo["SwapTotal"] - meminfo["SwapFree"] except (OSError, ValueError, IndexError): - pass + return {} + return found + + +def sample_memory() -> Dict[str, Any]: + """Cheap /proc snapshot (KiB): own RSS + MemTotal/MemAvailable + swap used. Linux-only + (``{}`` elsewhere), never raises; the 30s heartbeat embeds it so OOM cycles are classifiable.""" + sample = _proc_fields("/proc/self/status", {"VmRSS": "rss_kib"}) + mem = _proc_fields("/proc/meminfo", {"MemTotal": "mem_total_kib", "MemAvailable": "mem_available_kib", + "SwapTotal": "SwapTotal", "SwapFree": "SwapFree"}) + swap_total, swap_free = mem.pop("SwapTotal", None), mem.pop("SwapFree", None) + sample.update(mem) + if swap_total is not None and swap_free is not None: + sample["swap_used_kib"] = swap_total - swap_free return sample @@ -108,10 +83,10 @@ def _read_json(path: Path) -> Optional[Dict[str, Any]]: def _write_sentinel(payload: Dict[str, Any], home: Optional[Path]) -> None: - path = get_lifecycle_sentinel_path(home) try: from utils import atomic_json_write + path = get_lifecycle_sentinel_path(home) path.parent.mkdir(parents=True, exist_ok=True) atomic_json_write(path, payload, indent=None) except Exception: @@ -119,10 +94,9 @@ def _write_sentinel(payload: Dict[str, Any], home: Optional[Path]) -> None: def _append_exit_diag(record: Dict[str, Any], home: Optional[Path]) -> None: - """Append a JSON line to gateway-exit-diag.log (same format as the CLI's - ``_exit_diag`` records so existing tooling greps both).""" - path = _home_path(home, _EXIT_DIAG_RELATIVE) + """Append a JSON line to gateway-exit-diag.log (same format as the CLI's ``_exit_diag``).""" try: + path = _home_path(home, "logs", "gateway-exit-diag.log") path.parent.mkdir(parents=True, exist_ok=True) with path.open("a", encoding="utf-8") as fh: fh.write(json.dumps(record, default=str) + "\n") @@ -131,158 +105,118 @@ def _append_exit_diag(record: Dict[str, Any], home: Optional[Path]) -> None: def _pid_alive_with_start_time(pid: Any, start_time: Any) -> bool: - """True when ``pid`` is a live process matching ``start_time`` (±2s). - - Guards the takeover race: during ``--replace`` the old gateway can still - be mid-teardown when the new one boots — a live matching owner is a - planned handover, not an unclean death. - """ + """True when ``pid`` is a live process matching ``start_time`` (±2s) — guards the + ``--replace`` race: a live matching owner mid-teardown is a handover, not a death.""" try: pid_int = int(pid) - except (TypeError, ValueError): - return False - if pid_int <= 0: - return False - try: - # NOT os.kill(pid, 0): on Windows that sends CTRL_C_EVENT to the - # target's console group. _pid_exists is the canonical no-kill probe. - from gateway.status import _pid_exists + # NOT os.kill(pid, 0): on Windows that sends CTRL_C_EVENT to the target's console group. + from gateway.status import _pid_exists, get_process_start_time - if not _pid_exists(pid_int): + if pid_int <= 0 or not _pid_exists(pid_int): return False except Exception: return False - if start_time is None: - return True # alive; can't disambiguate PID reuse — err on "alive" try: - from gateway.status import get_process_start_time - - actual = get_process_start_time(pid_int) - return actual is None or abs(float(actual) - float(start_time)) <= 2.0 + actual = None if start_time is None else get_process_start_time(pid_int) + return actual is None or abs(float(actual) - float(start_time)) <= 2.0 # None: can't disambiguate PID reuse except Exception: return True +def _suspected_oom(mem: Dict[str, Any]) -> bool: + """Heuristic only (classification stays with the reader); thresholds are + memory_status' "critical" tier so a live warning and a post-mortem verdict agree.""" + from gateway.memory_status import _CRITICAL_AVAILABLE_FRACTION, _CRITICAL_AVAILABLE_KIB + + total, avail = mem.get("mem_total_kib"), mem.get("mem_available_kib") + if not isinstance(avail, int): + return False + return avail < _CRITICAL_AVAILABLE_KIB or ( + isinstance(total, int) and total > 0 and avail / total < _CRITICAL_AVAILABLE_FRACTION + ) + + def detect_unclean_exit(home: Optional[Path] = None) -> Optional[Dict[str, Any]]: - """Inspect the previous life's sentinel; return an evidence dict when it - died uncleanly, else ``None``. Read-only — does not rewrite the sentinel. - """ + """Evidence dict when the previous life died uncleanly, else ``None``. Read-only.""" sentinel = _read_json(get_lifecycle_sentinel_path(home)) if not sentinel or sentinel.get("phase") != "running": return None if _pid_alive_with_start_time(sentinel.get("pid"), sentinel.get("start_time")): return None # live owner — planned takeover in flight, not a death - evidence: Dict[str, Any] = { - "prior_pid": sentinel.get("pid"), - "prior_started_at": sentinel.get("started_at"), + "prior_pid": sentinel.get("pid"), "prior_started_at": sentinel.get("started_at"), "prior_start_time": sentinel.get("start_time"), } - # Enrich with the last heartbeat: last proven liveness and memory at that moment. try: from gateway.shutdown_watchdog import get_loop_heartbeat_path - hb = _read_json(get_loop_heartbeat_path(home)) + hb = _read_json(get_loop_heartbeat_path(home)) or {} except Exception: - hb = None + hb = {} if hb: evidence["last_heartbeat_at"] = hb.get("updated_at") - mem = hb.get("mem") - if isinstance(mem, dict): - evidence["last_heartbeat_mem"] = mem - total = mem.get("mem_total_kib") - avail = mem.get("mem_available_kib") - if isinstance(avail, int) and ( - avail < _LOW_MEM_AVAILABLE_KIB - or ( - isinstance(total, int) - and total > 0 - and avail / total < _LOW_MEM_AVAILABLE_FRACTION - ) - ): - evidence["suspected_oom"] = True + mem = hb.get("mem") + if isinstance(mem, dict): + evidence["last_heartbeat_mem"] = mem + if _suspected_oom(mem): + evidence["suspected_oom"] = True return evidence def check_state_db_integrity(home: Optional[Path] = None) -> str: - """Return ``"ok"``, ``"absent"``, or the first ``quick_check`` complaint. + """``"ok"``, ``"absent"``, or the first ``quick_check`` complaint. Never raises. - Called only after an unclean death — a SIGKILL mid-WAL-checkpoint can leave - half-written b-tree pages (see ``_enforce_macos_synchronous_full`` in - :mod:`hermes_state`). ``quick_check(1)`` stops at the first problem (~2s on - a healthy 500MB store): cheap once per unclean boot, too costly every boot. - Opened normally, not read-only: a WAL store needs its -shm sidecar for a - read-only open, and the PRAGMA itself writes nothing. Never raises. + Only after an unclean death — SIGKILL mid-WAL-checkpoint can leave half-written + b-tree pages. ``quick_check(1)`` stops at the first problem (~2s on a healthy + 500MB store): cheap once per unclean boot, too costly every boot. Opened + normally: a WAL store needs its -shm sidecar for read-only, and the PRAGMA writes nothing. """ - path = _home_path(home, _STATE_DB_RELATIVE) + path = _home_path(home, "state.db") if not path.exists(): return "absent" try: - conn = sqlite3.connect(str(path)) - try: + with closing(sqlite3.connect(str(path))) as conn: row = conn.execute("PRAGMA quick_check(1)").fetchone() - finally: - conn.close() - except Exception as exc: # sqlite3.Error, OSError, anything + except Exception as exc: return f"check-failed: {exc}" - if not row or row[0] is None: - return "check-failed: no result" - return str(row[0]) + return "check-failed: no result" if not row or row[0] is None else str(row[0]) + + +def _report_unclean_exit(evidence: Dict[str, Any], home: Optional[Path]) -> None: + """Integrity-check the store, persist the exit-diag record, log at WARNING.""" + # The death may have torn the store; this is the only moment we know to look. + verdict = evidence["state_db_integrity"] = check_state_db_integrity(home=home) + if verdict not in ("ok", "absent"): + logger.error( + "state.db FAILED integrity check after an unclean gateway exit: %s — sessions may read as " + "missing until it is repaired. Run `hermes doctor`.", + verdict, + ) + _append_exit_diag({"ts": _now_iso(), "tag": "gateway.previous_unclean_exit", "pid": os.getpid(), **evidence}, home) + logger.warning( + "Previous gateway life (pid=%s, started_at=%s) exited UNCLEANLY (no exit path ran — SIGKILL / OOM / " + "VM death). last_heartbeat_at=%s last_mem=%s suspected_oom=%s", + evidence.get("prior_pid"), evidence.get("prior_started_at"), evidence.get("last_heartbeat_at"), + evidence.get("last_heartbeat_mem"), evidence.get("suspected_oom", False), + ) def record_startup(home: Optional[Path] = None) -> Optional[Dict[str, Any]]: - """Boot-time entry point: report any unclean previous exit, then claim - the sentinel for the current life. - - Returns the unclean-exit evidence dict (also persisted to - ``gateway-exit-diag.log`` and logged at WARNING) or ``None``. Never - raises. - """ + """Boot entry point: report any unclean previous exit (evidence dict, also persisted + to ``gateway-exit-diag.log`` and logged at WARNING) then claim the sentinel. Never raises.""" evidence: Optional[Dict[str, Any]] = None try: evidence = detect_unclean_exit(home) if evidence is not None: - # The death may have torn the store; this is the only moment we know to look. - verdict = check_state_db_integrity(home=home) - evidence["state_db_integrity"] = verdict - if verdict not in ("ok", "absent"): - logger.error( - "state.db FAILED integrity check after an unclean gateway " - "exit: %s — sessions may read as missing until it is " - "repaired. Run `hermes doctor`.", - verdict, - ) - _append_exit_diag({ - "ts": datetime.now(timezone.utc).isoformat(), - "tag": "gateway.previous_unclean_exit", - "pid": os.getpid(), - **evidence, - }, home) - logger.warning( - "Previous gateway life (pid=%s, started_at=%s) exited UNCLEANLY " - "(no exit path ran — SIGKILL / OOM / VM death). " - "last_heartbeat_at=%s last_mem=%s suspected_oom=%s", - evidence.get("prior_pid"), - evidence.get("prior_started_at"), - evidence.get("last_heartbeat_at"), - evidence.get("last_heartbeat_mem"), - evidence.get("suspected_oom", False), - ) + _report_unclean_exit(evidence, home) except Exception: logger.debug("Unclean-exit detection failed", exc_info=True) - try: - claim: Dict[str, Any] = { - "phase": "running", - "pid": os.getpid(), - "start_time": time.time(), - "started_at": datetime.now(timezone.utc).isoformat(), - } - # Carry the verdict on the PREVIOUS life forward on the new sentinel: it - # is the only machine-readable copy (the exit-diag log is append-only - # prose) and /api/status reads it to report an OOM restart. Scoped to - # this life only — the next clean exit or boot rewrites the sentinel. + claim: Dict[str, Any] = {"phase": "running", "pid": os.getpid(), "start_time": time.time(), "started_at": _now_iso()} + # Carry the verdict on the PREVIOUS life on the new sentinel: it is the only + # machine-readable copy (/api/status reads it to report an OOM restart). + # Scoped to this life — the next clean exit or boot rewrites the sentinel. if evidence is not None: claim["prior_unclean_exit"] = True if evidence.get("suspected_oom"): @@ -293,49 +227,28 @@ def record_startup(home: Optional[Path] = None) -> Optional[Dict[str, Any]]: return evidence -def mark_exited( - exit_code: Optional[int] = None, - reason: str = "graceful_shutdown", - home: Optional[Path] = None, -) -> None: +def mark_exited(exit_code: Optional[int] = None, reason: str = "graceful_shutdown", home: Optional[Path] = None) -> None: """Mark the current life as cleanly exited. Idempotent, never raises. - Only rewrites the sentinel when it is provably owned by this process: - during a ``--replace`` takeover the replacement claims the sentinel before - the old process finishes teardown, and the old life must not clobber the - new owner's ``running`` phase. A sentinel with ``pid=None`` (or malformed) - has unknown ownership and is likewise left alone. + Only rewrites a sentinel provably owned by this process: during ``--replace`` + the replacement claims it before the old process finishes teardown and must not + be clobbered. ``pid=None`` / malformed sentinels have unknown ownership → left alone. """ try: sentinel = _read_json(get_lifecycle_sentinel_path(home)) if sentinel is not None and sentinel.get("pid") != os.getpid(): return - _write_sentinel({ - "phase": "exited", - "pid": os.getpid(), - "exit_code": exit_code, - "exit_reason": reason, - "exited_at": datetime.now(timezone.utc).isoformat(), - }, home) + _write_sentinel({"phase": "exited", "pid": os.getpid(), "exit_code": exit_code, "exit_reason": reason, + "exited_at": _now_iso()}, home) except Exception: logger.debug("Failed to mark lifecycle sentinel exited", exc_info=True) def read_prior_exit_label(profile_home: Path) -> str: - """Container-boot helper: ``clean`` / ``unclean`` / ``unknown`` summary of - how the profile's last gateway life ended. Read-only, exception-free; - used by ``hermes_cli.container_boot`` to annotate ``container-boot.log``. - """ + """``clean``/``unclean``/``unknown`` for the profile's last gateway life; annotates + ``container-boot.log``. ``running`` is unclean: the old PID namespace is gone at container boot.""" try: - sentinel = _read_json(get_lifecycle_sentinel_path(profile_home)) - if not sentinel: - return "unknown" - phase = sentinel.get("phase") - if phase == "exited": - return "clean" - if phase == "running": - # Old PID namespace is gone at container boot — never exited cleanly. - return "unclean" + phase = (_read_json(get_lifecycle_sentinel_path(profile_home)) or {}).get("phase") + return {"exited": "clean", "running": "unclean"}.get(phase, "unknown") except Exception: - pass - return "unknown" + return "unknown" diff --git a/gateway/media_policy.py b/gateway/media_policy.py index 2d6019a34d..1440549b32 100644 --- a/gateway/media_policy.py +++ b/gateway/media_policy.py @@ -1,21 +1,12 @@ """Shared config→env bridge for media-delivery policy. -``validate_media_delivery_path`` (gateway/platforms/base.py) reads its policy -from environment variables: - - - ``HERMES_MEDIA_DELIVERY_STRICT`` <- gateway.strict - - ``HERMES_MEDIA_ALLOW_DIRS`` <- gateway.media_delivery_allow_dirs - - ``HERMES_MEDIA_TRUST_RECENT_FILES`` <- gateway.trust_recent_files - -The translation used to run only in gateway startup, so standalone delivery -paths (``hermes cron run``, ``hermes send``, a standalone cron tick) filtered -MEDIA paths under a different policy and silently dropped attachments in -strict/allowlisted deployments (text is unaffected -- only media goes through -path validation). ``apply_media_policy_env()`` is the shared, -idempotent helper every delivery entrypoint calls before filtering media paths. - -Precedence: an explicitly-set environment variable WINS over config.yaml, so a -shell-exported override (and gateway startup's own earlier run) survives. +``validate_media_delivery_path`` reads ``HERMES_MEDIA_DELIVERY_STRICT`` (gateway.strict), +``HERMES_MEDIA_ALLOW_DIRS`` (gateway.media_delivery_allow_dirs) and +``HERMES_MEDIA_TRUST_RECENT_FILES`` (gateway.trust_recent_files). Every delivery +entrypoint (gateway startup, ``hermes cron run``, ``hermes send``) calls +:func:`apply_media_policy_env` first so standalone paths filter under the gateway's +policy instead of silently dropping attachments in strict/allowlisted deployments. +An explicitly-set env var WINS over config.yaml, so shell overrides survive. """ from __future__ import annotations @@ -26,9 +17,8 @@ from typing import Any, Dict, Optional logger = logging.getLogger(__name__) -_STRICT_ENV = "HERMES_MEDIA_DELIVERY_STRICT" +_FLAG_ENVS = (("strict", "HERMES_MEDIA_DELIVERY_STRICT"), ("trust_recent_files", "HERMES_MEDIA_TRUST_RECENT_FILES")) _ALLOW_DIRS_ENV = "HERMES_MEDIA_ALLOW_DIRS" -_TRUST_RECENT_ENV = "HERMES_MEDIA_TRUST_RECENT_FILES" def _load_gateway_cfg(config: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: @@ -50,25 +40,19 @@ def _set_env_default(env: str, value: str) -> None: def _allow_dirs_str(allow_dirs: Any) -> str: - if isinstance(allow_dirs, str): - return allow_dirs if isinstance(allow_dirs, (list, tuple)): return os.pathsep.join(str(p) for p in allow_dirs if p) - return "" + return allow_dirs if isinstance(allow_dirs, str) else "" def apply_media_policy_env(config: Optional[Dict[str, Any]] = None) -> None: - """Bridge gateway media-policy settings from config.yaml into the env. - - Idempotent and env-wins: a variable already present is never overwritten. - Never raises — a policy-bridge failure must not break delivery; the - validator falls back to its defaults exactly as before. - """ + """Bridge gateway media-policy settings from config.yaml into the env. Idempotent, + env-wins, never raises — a bridge failure must not break delivery (validator defaults apply).""" try: gateway_cfg = _load_gateway_cfg(config) if not gateway_cfg: return - for key, env in (("strict", _STRICT_ENV), ("trust_recent_files", _TRUST_RECENT_ENV)): + for key, env in _FLAG_ENVS: flag = gateway_cfg.get(key) if flag is not None: _set_env_default(env, "1" if flag else "0") diff --git a/gateway/media_repair.py b/gateway/media_repair.py index adfc3c3bdc..9090b2b859 100644 --- a/gateway/media_repair.py +++ b/gateway/media_repair.py @@ -1,19 +1,11 @@ """Repair model-mangled ``computer_use`` screenshot paths in final responses. -``computer_use`` persists a screenshot into the Hermes image cache and tells the -model its absolute path. Some models rewrite a Windows path into a POSIX-looking -one (``C:\\Users\\Alice\\...`` -> ``/Users/Alice/...``) inside an explicit -``MEDIA:`` directive, so delivery-path validation rejects it and drops the -attachment. - -The repair is deliberately narrow: it only rewrites paths inside a response that -*already* carries an explicit ``MEDIA:`` directive, and only when the directive's -generated ``computer_use_`` basename exactly matches a canonical screenshot -path returned by ``computer_use`` in the current turn. It never auto-attaches -captures; normal media path validation still runs afterwards. - -Own module (like ``gateway/media_policy.py``) so the gateway turn path, gateway -background tasks and cron delivery share one implementation. +Some models rewrite the Windows path ``computer_use`` reported into a POSIX-looking +one (``C:\\Users\\Alice\\...`` -> ``/Users/Alice/...``) inside an explicit ``MEDIA:`` +directive, so delivery-path validation rejects it and drops the attachment. +Deliberately narrow: only rewrites paths in a response that *already* carries a +``MEDIA:`` directive whose ``computer_use_`` basename exactly matches a +canonical path returned this turn. Never auto-attaches; validation still runs. """ from __future__ import annotations @@ -29,15 +21,12 @@ logger = logging.getLogger(__name__) # root, or UNC share. Shared string so the summary regex stays in sync. _ABS_PATH_PREFIX_PATTERN = r"(?:[A-Za-z]:[/\\]|/|\\\\)" _ABS_PATH_PREFIX_RE = re.compile(r"^" + _ABS_PATH_PREFIX_PATTERN) +_CAPTURE_BASENAME_PATTERN = r"computer_use_[0-9a-f]{32}\.(?:png|jpe?g)" -_COMPUTER_USE_CAPTURE_BASENAME_RE = re.compile( - r"^computer_use_[0-9a-f]{32}\.(?:png|jpe?g)$", - re.IGNORECASE, -) +_COMPUTER_USE_CAPTURE_BASENAME_RE = re.compile(r"^" + _CAPTURE_BASENAME_PATTERN + r"$", re.IGNORECASE) _COMPUTER_USE_CAPTURE_SUMMARY_RE = re.compile( r"\(shareable screenshot saved to " - r"(?P" + _ABS_PATH_PREFIX_PATTERN + r"[^\r\n]*?" - r"computer_use_[0-9a-f]{32}\.(?:png|jpe?g))\)", + r"(?P" + _ABS_PATH_PREFIX_PATTERN + r"[^\r\n]*?" + _CAPTURE_BASENAME_PATTERN + r")\)", re.IGNORECASE, ) @@ -50,8 +39,7 @@ def tool_name_by_call_id(messages: List[Dict[str, Any]]) -> Dict[str, str]: continue for call in msg.get("tool_calls") or []: call_id = call.get("id") or call.get("call_id") - fn = call.get("function") or {} - name = str(fn.get("name") or call.get("name") or "") + name = str((call.get("function") or {}).get("name") or call.get("name") or "") if call_id and name: mapping[str(call_id)] = name return mapping @@ -59,130 +47,96 @@ def tool_name_by_call_id(messages: List[Dict[str, Any]]) -> Dict[str, str]: def _computer_use_capture_basename(path: Any) -> str: """Canonical (lowercased) capture basename for either separator style, or ''.""" - value = str(path or "").strip().strip("`\"'") - basename = re.split(r"[/\\]", value)[-1] + basename = re.split(r"[/\\]", str(path or "").strip().strip("`\"'"))[-1] return basename.lower() if _COMPUTER_USE_CAPTURE_BASENAME_RE.fullmatch(basename) else "" def _iter_computer_use_capture_paths(content: Any) -> Iterator[str]: - """Yield persisted screenshot paths from computer_use result content. - - The tool can return JSON, a multimodal content list, or a text fallback; the - latter two keep the canonical path in the human-readable summary even though - the multimodal envelope's ``meta`` is not stored in the tool message. - """ + """Yield persisted screenshot paths from computer_use result content (JSON, a + multimodal list, or text; the latter two keep the canonical path in the summary + line since the envelope's ``meta`` is not stored in the tool message).""" if isinstance(content, str): stripped = content.strip() - if stripped.startswith(("{", "[")): - # Parse JSON first, never regex-scan it: JSON escaping doubles - # backslashes, so a raw-text hit would yield a path that exists - # nowhere. Fail closed on unparseable (truncated) JSON. - try: - payload = json.loads(stripped) - except Exception: - return - if isinstance(payload, (dict, list)): - yield from _iter_computer_use_capture_paths(payload) + if not stripped.startswith(("{", "[")): + for match in _COMPUTER_USE_CAPTURE_SUMMARY_RE.finditer(content): + yield match.group("path").strip() return - for match in _COMPUTER_USE_CAPTURE_SUMMARY_RE.finditer(content): - yield match.group("path").strip() - return - - if isinstance(content, list): + # Parse JSON, never regex-scan it: JSON escaping doubles backslashes, so a raw-text + # hit would yield a path that exists nowhere. Fail closed on truncated JSON. + try: + payload = json.loads(stripped) + except Exception: + return + if isinstance(payload, (dict, list)): + yield from _iter_computer_use_capture_paths(payload) + elif isinstance(content, list): for part in content: yield from _iter_computer_use_capture_paths(part) - return - - if not isinstance(content, dict): - return - - screenshot_path = content.get("screenshot_path") - if isinstance(screenshot_path, str): - yield screenshot_path - meta = content.get("meta") - if isinstance(meta, dict) and isinstance(meta.get("screenshot_path"), str): - yield meta["screenshot_path"] - # Producer shapes (tools/computer_use/tool.py::_capture_response): - # content/text = multimodal parts; text_summary/summary = the line carrying - # "(shareable screenshot saved to ...)". - for field in ("content", "text", "text_summary", "summary"): - nested = content.get(field) - if isinstance(nested, (str, dict, list)): - yield from _iter_computer_use_capture_paths(nested) + elif isinstance(content, dict): + meta = content.get("meta") + for holder in (content, meta if isinstance(meta, dict) else {}): + if isinstance(holder.get("screenshot_path"), str): + yield holder["screenshot_path"] + # Producer shapes (computer_use ``_capture_response``): content/text = multimodal + # parts; text_summary/summary = the "(shareable screenshot saved to ...)" line. + for field in ("content", "text", "text_summary", "summary"): + nested = content.get(field) + if isinstance(nested, (str, dict, list)): + yield from _iter_computer_use_capture_paths(nested) def _current_turn_messages(messages: List[Dict[str, Any]], history_offset: int) -> List[Dict[str, Any]]: - if not history_offset: - return messages - if len(messages) >= history_offset: + if not history_offset or len(messages) >= history_offset: return messages[history_offset:] - # Compression can invalidate the slice boundary: recover the turn from its - # last user message, fail closed if none remains. Deliberately narrower than - # gateway/run.py::_collect_auto_append_media_tags' scan-everything fallback — - # that decides whether to ATTACH; this only rewrites paths already emitted. + # Compression can invalidate the slice boundary: recover the turn from its last user + # message, fail closed if none remains. Narrower than run.py's scan-everything fallback + # for auto-attach — that decides whether to ATTACH; this only rewrites emitted paths. for index in range(len(messages) - 1, -1, -1): if messages[index].get("role") == "user": return messages[index:] return [] -def repair_explicit_computer_use_media_paths( - response: str, - messages: List[Dict[str, Any]], - history_offset: int = 0, -) -> str: - """Recover model-mangled paths for explicitly requested screenshots. - - Repairs only an already-explicit ``MEDIA:`` directive whose generated - basename case-insensitively matches a canonical screenshot path from this - turn. Fail-open: the repair is cosmetic, so any unexpected error returns - the response unchanged rather than aborting delivery. - """ +def repair_explicit_computer_use_media_paths(response: str, messages: List[Dict[str, Any]], history_offset: int = 0) -> str: + """Recover model-mangled paths in explicit ``MEDIA:`` directives whose basename + matches (case-insensitively) a canonical screenshot path from this turn. + Fail-open: the repair is cosmetic, so any error returns the response unchanged.""" try: - return _repair_explicit_computer_use_media_paths_inner( - response, messages, history_offset - ) + return _repair_explicit_computer_use_media_paths_inner(response, messages, history_offset) except Exception: logger.debug("computer_use media path repair failed", exc_info=True) return response -def _repair_explicit_computer_use_media_paths_inner( - response: str, - messages: List[Dict[str, Any]], - history_offset: int = 0, -) -> str: - if "MEDIA:" not in response: - return response - - turn_messages = _current_turn_messages(messages, history_offset) +def _canonical_capture_paths(turn_messages: List[Dict[str, Any]]) -> Dict[str, str]: + """``{lowercase basename: absolute canonical path}`` from this turn's computer_use results.""" call_id_names = tool_name_by_call_id(turn_messages) - - canonical_by_basename: Dict[str, str] = {} + canonical: Dict[str, str] = {} for msg in turn_messages: if msg.get("role") not in {"tool", "function"}: continue call_id = str(msg.get("tool_call_id") or msg.get("call_id") or "") - tool_name = str( - msg.get("name") or msg.get("tool_name") or call_id_names.get(call_id) or "" - ) - if tool_name != "computer_use": + if str(msg.get("name") or msg.get("tool_name") or call_id_names.get(call_id) or "") != "computer_use": continue for path in _iter_computer_use_capture_paths(msg.get("content")): basename = _computer_use_capture_basename(path) if basename and _ABS_PATH_PREFIX_RE.match(path): - canonical_by_basename[basename] = path + canonical[basename] = path + return canonical + +def _repair_explicit_computer_use_media_paths_inner(response: str, messages: List[Dict[str, Any]], history_offset: int = 0) -> str: + if "MEDIA:" not in response: + return response + canonical_by_basename = _canonical_capture_paths(_current_turn_messages(messages, history_offset)) if not canonical_by_basename: return response - # Lazy: keeps `import gateway.media_repair` cheap for standalone cron - # processes that may never hit a MEDIA: response. No import cycle either way. + # Lazy: keeps the import cheap for standalone cron processes that may never hit MEDIA:. from gateway.platforms.base import BasePlatformAdapter - media_files, _ = BasePlatformAdapter.extract_media(response) repaired = response - for emitted_path, _is_voice in media_files: + for emitted_path, _is_voice in BasePlatformAdapter.extract_media(response)[0]: canonical = canonical_by_basename.get(_computer_use_capture_basename(emitted_path)) if canonical and emitted_path != canonical: repaired = repaired.replace(emitted_path, canonical) diff --git a/gateway/memory_monitor.py b/gateway/memory_monitor.py index 3ac51b531e..b215404882 100644 --- a/gateway/memory_monitor.py +++ b/gateway/memory_monitor.py @@ -1,15 +1,15 @@ """Periodic process memory usage logging for the gateway. -Ported from cline/cline#10343 (src/standalone/memory-monitor.ts). Emits one -grep-friendly ``[MEMORY] ...`` line every N seconds (default 300) from a daemon -thread so slow leaks in the long-lived gateway show up as an RSS time series in -``agent.log`` / ``gateway.log``. A baseline snapshot is logged on start and a -final one on stop. Uses stdlib ``resource`` first, ``psutil`` as fallback -(Windows); if neither works the monitor warns once and stays disabled. +Emits one grep-friendly ``[MEMORY] ...`` line every N seconds (default 300) +from a daemon thread so slow leaks show up as an RSS time series in the logs. +A baseline snapshot is logged on start and a final one on stop. Uses stdlib +``resource`` first, ``psutil`` as fallback (Windows); if neither works the +monitor warns once and stays disabled. """ from __future__ import annotations +import contextlib import gc import logging import os @@ -17,7 +17,6 @@ import sys import threading import time from typing import Optional -import contextlib logger = logging.getLogger(__name__) @@ -26,7 +25,6 @@ _BYTES_TO_MB = 1024 * 1024 _monitor_thread: Optional[threading.Thread] = None _stop_event: Optional[threading.Event] = None _start_time: Optional[float] = None -_interval_seconds: float = 300.0 _lock = threading.Lock() @@ -36,8 +34,7 @@ def _get_rss_mb() -> Optional[int]: import resource # ru_maxrss is KB on Linux but bytes on macOS. - maxrss = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss - return int(maxrss / (_BYTES_TO_MB if sys.platform == "darwin" else 1024)) + return int(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / (_BYTES_TO_MB if sys.platform == "darwin" else 1024)) except Exception: pass try: @@ -49,25 +46,12 @@ def _get_rss_mb() -> Optional[int]: def log_memory_usage(prefix: str = "") -> None: - """Log current memory usage as ``[MEMORY] [ ]rss=... gc=... threads=... uptime=...``. - - Safe to call on-demand from any thread at lifecycle moments. - """ + """Log ``[MEMORY] [ ]rss=... gc=... threads=... uptime=...``; safe from any thread.""" rss = _get_rss_mb() - uptime = int(time.monotonic() - _start_time) if _start_time else 0 - try: - gc_counts = gc.get_count() # (gen0, gen1, gen2) - except Exception: - gc_counts = (0, 0, 0) - try: - thread_count = threading.active_count() - except Exception: - thread_count = 0 - - tag = f"{prefix} " if prefix else "" - rss_text = "unavailable" if rss is None else f"{rss}MB" logger.info( - "[MEMORY] %srss=%s gc=%s threads=%d uptime=%ds", tag, rss_text, gc_counts, thread_count, uptime + "[MEMORY] %srss=%s gc=%s threads=%d uptime=%ds", f"{prefix} " if prefix else "", + "unavailable" if rss is None else f"{rss}MB", gc.get_count(), threading.active_count(), + int(time.monotonic() - _start_time) if _start_time else 0, ) @@ -76,41 +60,32 @@ def _monitor_loop(stop_event: threading.Event, interval: float) -> None: while not stop_event.wait(interval): try: log_memory_usage() - except Exception as e: - # Never let the monitor crash the gateway. + except Exception as e: # never let the monitor crash the gateway logger.debug("Memory monitor iteration failed: %s", e) def start_memory_monitoring(interval_seconds: float = 300.0) -> bool: - """Start periodic memory logging in a daemon thread (baseline logged immediately). - - Returns True if a fresh monitor was started; False if one is already running - or RSS introspection is unavailable (warned once). - """ - global _monitor_thread, _stop_event, _start_time, _interval_seconds + """Start periodic logging in a daemon thread (baseline logged immediately). False if + already running or RSS introspection is unavailable (warned once).""" + global _monitor_thread, _stop_event, _start_time with _lock: if _monitor_thread is not None and _monitor_thread.is_alive(): return False if _get_rss_mb() is None: logger.warning( - "[MEMORY] Memory monitoring unavailable: neither resource.getrusage " - "nor psutil could read process RSS — skipping periodic logging.", + "[MEMORY] Memory monitoring unavailable: neither resource.getrusage nor psutil could read process RSS " + "— skipping periodic logging.", ) return False - _start_time = time.monotonic() - _interval_seconds = float(interval_seconds) _stop_event = threading.Event() log_memory_usage(prefix="baseline") _monitor_thread = threading.Thread( - target=_monitor_loop, - args=(_stop_event, _interval_seconds), - name="gateway-memory-monitor", - daemon=True, + target=_monitor_loop, args=(_stop_event, float(interval_seconds)), name="gateway-memory-monitor", daemon=True ) _monitor_thread.start() - logger.info("[MEMORY] Periodic memory monitoring started (interval: %ds)", int(_interval_seconds)) + logger.info("[MEMORY] Periodic memory monitoring started (interval: %ds)", int(interval_seconds)) return True @@ -124,14 +99,9 @@ def stop_memory_monitoring(timeout: float = 2.0) -> None: with contextlib.suppress(Exception): log_memory_usage(prefix="shutdown") _stop_event.set() - thread = _monitor_thread - _monitor_thread = None - _stop_event = None - - # Join outside the lock so a stuck log call can't deadlock shutdown. - with contextlib.suppress(Exception): + thread, _monitor_thread, _stop_event = _monitor_thread, None, None + with contextlib.suppress(Exception): # join outside the lock so a stuck log call can't deadlock the stop path thread.join(timeout=timeout) - logger.info("[MEMORY] Periodic memory monitoring stopped") diff --git a/gateway/memory_status.py b/gateway/memory_status.py index 9170083346..e6a4600a64 100644 --- a/gateway/memory_status.py +++ b/gateway/memory_status.py @@ -1,104 +1,74 @@ """Memory status rollup for ``/api/status``. -Read side for memory-pressure signals the gateway already persists but only -logs: the 30s ``state/gateway.heartbeat`` (RSS + MemAvailable/MemTotal + swap, -from :func:`gateway.shutdown_watchdog.write_loop_heartbeat`) and the lifecycle -sentinel's ``suspected_oom`` flag (:func:`gateway.lifecycle_ledger.record_startup`). -Distills them into a compact block the dashboard SPA and the NAS availability -sweep consume — no new sampling, no IPC, two small file reads. - -Public-safety: ``/api/status`` is unauthenticated (``PUBLIC_API_PATHS``), so this -block carries only coarse numbers (MB granularity), enums, and booleans — the -same disclosure class as ``active_agents`` / ``nous_session_valid``. - -Best-effort and read-only: a missing/corrupt file degrades to +Read side for signals the gateway already persists: the 30s ``state/gateway.heartbeat`` +(RSS + MemAvailable/MemTotal + swap) and the lifecycle sentinel's ``suspected_oom`` +flag — two small file reads, no IPC. ``/api/status`` is unauthenticated, so only +coarse numbers (MB), enums and booleans. A missing/corrupt file degrades to ``pressure="unknown"`` rather than raising into the status endpoint. """ from __future__ import annotations -import logging from datetime import datetime, timezone from pathlib import Path from typing import Any, Dict, Optional -logger = logging.getLogger(__name__) - -# Thresholds on system MemAvailable. ``critical`` mirrors the lifecycle ledger's -# OOM-suspicion heuristics (_LOW_MEM_AVAILABLE_KIB / _LOW_MEM_AVAILABLE_FRACTION): -# a level that would make a later unclean death "suspected OOM" should already -# warn while the process is alive. +# Thresholds on system MemAvailable. ``critical`` doubles as the lifecycle +# ledger's OOM-suspicion heuristic: a level that makes a later unclean death +# "suspected OOM" already warns while the process is alive. _CRITICAL_AVAILABLE_KIB = 64 * 1024 # < 64 MiB available _CRITICAL_AVAILABLE_FRACTION = 0.05 # < 5% of MemTotal _ELEVATED_AVAILABLE_KIB = 128 * 1024 # < 128 MiB available _ELEVATED_AVAILABLE_FRACTION = 0.15 # < 15% of MemTotal +_PRESSURE_TIERS = ( # order-sensitive: worst first + ("critical", _CRITICAL_AVAILABLE_KIB, _CRITICAL_AVAILABLE_FRACTION), + ("elevated", _ELEVATED_AVAILABLE_KIB, _ELEVATED_AVAILABLE_FRACTION), +) -# Writer cadence is 30s (DEFAULT_HEARTBEAT_INTERVAL_S); 150s tolerates a briefly -# stalled loop without letting a long-dead gateway's last sample pose as current. +# Writer cadence is 30s; 150s tolerates a briefly stalled loop without letting +# a long-dead gateway's last sample pose as current. _HEARTBEAT_FRESH_TTL_S = 150.0 -_KIB_PER_MB = 1024 - def _nonneg_int(value: Any) -> Optional[int]: """Return *value* if it is a non-negative int (bools rejected), else None.""" - if isinstance(value, bool) or not isinstance(value, int) or value < 0: - return None - return value + return value if isinstance(value, int) and not isinstance(value, bool) and value >= 0 else None def _mb(kib: Any) -> Optional[int]: - kib = _nonneg_int(kib) - return None if kib is None else kib // _KIB_PER_MB + return None if _nonneg_int(kib) is None else kib // 1024 def _parse_iso(value: Any) -> Optional[datetime]: - if not isinstance(value, str) or not value: - return None try: - parsed = datetime.fromisoformat(value) + parsed = datetime.fromisoformat(value) if isinstance(value, str) and value else None except ValueError: return None - return parsed if parsed.tzinfo is not None else parsed.replace(tzinfo=timezone.utc) + return parsed.replace(tzinfo=timezone.utc) if parsed is not None and parsed.tzinfo is None else parsed def classify_pressure(available_kib: Any, total_kib: Any) -> str: - """Map a MemAvailable/MemTotal pair to ``ok``/``elevated``/``critical``. - - ``unknown`` when the sample is missing or malformed — the caller must - not treat "we could not read it" as "memory is fine". - """ - available = _nonneg_int(available_kib) + """``ok``/``elevated``/``critical`` from MemAvailable/MemTotal; ``unknown`` when the + sample is missing/malformed — "could not read it" must never read as "fine".""" + available, total = _nonneg_int(available_kib), _nonneg_int(total_kib) if available is None: return "unknown" - total = _nonneg_int(total_kib) fraction = available / total if total else None - for level, kib_floor, frac_floor in ( - ("critical", _CRITICAL_AVAILABLE_KIB, _CRITICAL_AVAILABLE_FRACTION), - ("elevated", _ELEVATED_AVAILABLE_KIB, _ELEVATED_AVAILABLE_FRACTION), - ): + for level, kib_floor, frac_floor in _PRESSURE_TIERS: if available < kib_floor or (fraction is not None and fraction < frac_floor): return level return "ok" -def _read_heartbeat(home: Optional[Path]) -> Optional[Dict[str, Any]]: - try: - from gateway.lifecycle_ledger import _read_json - from gateway.shutdown_watchdog import get_loop_heartbeat_path - - return _read_json(get_loop_heartbeat_path(home)) - except Exception: - return None - - -def _read_sentinel(home: Optional[Path]) -> Optional[Dict[str, Any]]: +def _read_state_files(home: Optional[Path]) -> tuple: + """``(heartbeat, sentinel)`` dicts, each ``None`` when unreadable.""" try: from gateway.lifecycle_ledger import _read_json, get_lifecycle_sentinel_path + from gateway.shutdown_watchdog import get_loop_heartbeat_path - return _read_json(get_lifecycle_sentinel_path(home)) + return _read_json(get_loop_heartbeat_path(home)), _read_json(get_lifecycle_sentinel_path(home)) except Exception: - return None + return None, None def collect_memory_status( @@ -106,38 +76,25 @@ def collect_memory_status( *, now: Optional[datetime] = None, ) -> Dict[str, Any]: - """Build the ``memory`` block for ``/api/status``. - - ``home`` scopes the read to a profile's HERMES_HOME (``None`` = active - profile); ``now`` is injectable for tests. Always returns a dict and never - raises — a down/never-started gateway or corrupt files yield - ``{"pressure": "unknown", ...}`` plus whatever fields could be recovered. - """ + """``memory`` block for ``/api/status``; ``home`` scopes to a profile (``None`` = + active), ``now`` is injectable. Never raises — a down gateway or corrupt files + yield ``pressure="unknown"`` plus whatever fields could be recovered.""" moment = now or datetime.now(timezone.utc) status: Dict[str, Any] = { - "pressure": "unknown", - "gateway_rss_mb": None, - "system_total_mb": None, - "system_available_mb": None, - "swap_used_mb": None, - "sampled_at": None, - "last_boot_unclean": False, - "last_boot_suspected_oom": False, - # Identity of the CURRENT gateway life (sentinel started_at) — changes on - # every boot, so the dashboard can key banner dismissal on it and - # acknowledging one OOM restart does not mute the NEXT one. + "pressure": "unknown", "gateway_rss_mb": None, "system_total_mb": None, "system_available_mb": None, + "swap_used_mb": None, "sampled_at": None, "last_boot_unclean": False, "last_boot_suspected_oom": False, + # Identity of the CURRENT life (sentinel started_at): the dashboard keys + # banner dismissal on it so acknowledging one OOM restart does not mute the NEXT. "boot_id": None, } - heartbeat = _read_heartbeat(home) + heartbeat, sentinel = _read_state_files(home) if heartbeat: - sampled_at = _parse_iso(heartbeat.get("updated_at")) - mem = heartbeat.get("mem") + sampled_at, mem = _parse_iso(heartbeat.get("updated_at")), heartbeat.get("mem") if isinstance(mem, dict): - status["gateway_rss_mb"] = _mb(mem.get("rss_kib")) - status["system_total_mb"] = _mb(mem.get("mem_total_kib")) - status["system_available_mb"] = _mb(mem.get("mem_available_kib")) - status["swap_used_mb"] = _mb(mem.get("swap_used_kib")) + for dst, src in (("gateway_rss_mb", "rss_kib"), ("system_total_mb", "mem_total_kib"), + ("system_available_mb", "mem_available_kib"), ("swap_used_mb", "swap_used_kib")): + status[dst] = _mb(mem.get(src)) if sampled_at is not None: status["sampled_at"] = sampled_at.isoformat() # Stale sample: numbers still reported (sampled_at says when) but @@ -146,12 +103,10 @@ def collect_memory_status( if 0 <= (moment - sampled_at).total_seconds() <= _HEARTBEAT_FRESH_TTL_S: status["pressure"] = classify_pressure(mem.get("mem_available_kib"), mem.get("mem_total_kib")) - sentinel = _read_sentinel(home) if sentinel: status["last_boot_unclean"] = bool(sentinel.get("prior_unclean_exit")) status["last_boot_suspected_oom"] = bool(sentinel.get("prior_suspected_oom")) started_at = sentinel.get("started_at") - if isinstance(started_at, str) and started_at: - status["boot_id"] = started_at + status["boot_id"] = started_at if isinstance(started_at, str) and started_at else None return status diff --git a/gateway/message_timestamps.py b/gateway/message_timestamps.py index 4bb5fe4579..9f353b60bb 100644 --- a/gateway/message_timestamps.py +++ b/gateway/message_timestamps.py @@ -12,17 +12,19 @@ from datetime import datetime from typing import Any, Optional, Tuple -# Current gateway format: [Tue 2026-04-28 13:40:53 CEST] -_HUMAN_TIMESTAMP_RE = re.compile( - r"^\[(?P[A-Z][a-z]{2}) " +# Leading timestamp prefix, either the current human format +# ``[Tue 2026-04-28 13:40:53 CEST]`` or the older ISO one +# ``[2026-04-13T17:02:06+0200]`` / ``[...+02:00]`` (human tried first). +_TIMESTAMP_PREFIX_RE = re.compile( + r"^\[(?:" + r"(?P[A-Z][a-z]{2}) " r"(?P\d{4}-\d{2}-\d{2}) " r"(?P