ouyangbo/hermes-agent
1
0
Fork 0
Code Issues Pull Requests Actions 11 Packages Projects Releases Wiki Activity

Merge branch 'simp/r3-10' into simp/integration3

Browse Source
This commit is contained in:
Teknium
2026-09-02 21:57:15 -07:00
parent 63e9553fa2 82f2fb9029
commit 2e4a42605e
50 changed files with 5229 additions and 10785 deletions
Download Patch File Download Diff File Expand all files Collapse all files
gateway/__init__.py
+5 -23
View File
@@ -1,13 +1,5 @@
"""
Hermes Gateway - Multi-platform messaging integration.
This module provides a unified gateway for connecting the Hermes agent
to various messaging platforms (Telegram, Discord, WhatsApp, Weixin, and more) with:
- Session management (persistent conversations with reset policies)
- Dynamic context injection (agent knows where messages come from)
- Delivery routing (cron job outputs to appropriate channels)
- Platform-specific toolsets (different capabilities per platform)
"""
"""Hermes Gateway - multi-platform messaging integration (sessions, context
injection, delivery routing, platform-specific toolsets)."""
from .config import GatewayConfig, PlatformConfig, HomeChannel, load_gateway_config
from .session import (
@@ -19,17 +11,7 @@ from .session import (
from .delivery import DeliveryRouter, DeliveryTarget
__all__ = [
# Config
"GatewayConfig",
"PlatformConfig",
"HomeChannel",
"load_gateway_config",
# Session
"SessionContext",
"SessionStore",
"SessionResetPolicy",
"build_session_context_prompt",
# Delivery
"DeliveryRouter",
"DeliveryTarget",
"GatewayConfig", "PlatformConfig", "HomeChannel", "load_gateway_config",
"SessionContext", "SessionStore", "SessionResetPolicy", "build_session_context_prompt",
"DeliveryRouter", "DeliveryTarget",
]
gateway/agent_cache_pressure.py
+65 -141
View File
@@ -1,15 +1,11 @@
"""Memory-pressure bounds for the gateway's per-session AIAgent cache.
Each cached ``AIAgent`` pins ``_session_messages`` (the full live transcript,
tool outputs included — tens of MB on a tool-heavy session). The cache's LRU
cap counts entries, not bytes, and the idle TTL defers eviction for busy
sessions, so neither sees actual memory use. This module supplies that signal:
the process's own anonymous RSS against a budget derived from the cgroup limit.
``GatewayRunner._sweep_agent_cache_under_pressure`` uses it to shed LRU
transcripts via the soft-eviction path (rebuilt from the persisted session on
the next turn).
Everything here is pure or read-only. Config lives under ``agent.agent_cache``.
Each cached ``AIAgent`` pins its full live transcript (tens of MB on a tool-heavy
session); the LRU cap counts entries, not bytes, and the idle TTL defers eviction
for busy sessions, so neither sees actual memory use. This module supplies that
signal — own anonymous RSS against a budget derived from the cgroup limit — and
``GatewayRunner`` sheds LRU transcripts via soft eviction (rebuilt from the
persisted session next turn). Pure/read-only; config under ``agent.agent_cache``.
"""
from __future__ import annotations
@@ -20,29 +16,23 @@ from dataclasses import dataclass
from pathlib import Path
from typing import Any, Callable, Iterable, List, Optional, Tuple
# Shed well under the limit: once cgroup ``memory.high`` throttling kicks in
# (swap full), a SIGTERM flush cannot finish inside systemd's stop timeout.
# Shed well under the limit: once cgroup ``memory.high`` throttling kicks in (swap full),
# a SIGTERM flush cannot finish inside systemd's stop timeout.
_AUTO_BUDGET_FRACTION = 0.65
# Below this a budget is noise — small containers would evict every pass and
# never keep a warm prefix.
# Below this a budget is noise — small containers would evict every pass and never keep a warm prefix.
_AUTO_BUDGET_FLOOR_MB = 512
_DEFAULT_MAX_EVICTIONS_PER_PASS = 16
# Never shed the hottest sessions: their prompt cache is worth the most, and
# evicting them just moves the cost to the next turn.
# Never shed the hottest sessions: their prompt cache is worth the most; evicting them
# just moves the cost to the next turn.
_DEFAULT_PROTECT_RECENT = 8
_BYTES_PER_MB = 1024 * 1024
_OFF_WORDS = frozenset({"", "off", "none", "false", "disabled"})
@dataclass(frozen=True)
class AgentCacheBounds:
"""Operator-facing bounds for the per-session agent cache.
``max_size`` / ``idle_ttl_secs`` are ``None`` when unset so ``gateway/run.py``
keeps its module defaults; ``memory_high_mb`` is ``None`` when pressure
eviction is off.
"""
"""Operator-facing bounds. ``max_size``/``idle_ttl_secs`` are ``None`` when unset
so ``gateway/run.py`` keeps its defaults; ``memory_high_mb`` ``None`` = pressure eviction off."""
max_size: Optional[int] = None
idle_ttl_secs: Optional[float] = None
@@ -51,64 +41,43 @@ class AgentCacheBounds:
protect_recent: int = _DEFAULT_PROTECT_RECENT
def _positive(value: Any, cast: Callable[[Any], Any]) -> Any:
def _is_int(value: Any) -> bool:
return isinstance(value, int) and not isinstance(value, bool)
def _positive(value: Any, cast: Callable[[Any], Any] = int) -> Any:
"""``cast(value)`` if it is a positive number (bools rejected), else None."""
if isinstance(value, bool) or value is None:
return None
try:
parsed = cast(value)
parsed = None if isinstance(value, bool) or value is None else cast(value)
except (TypeError, ValueError):
return None
return parsed if parsed > 0 else None
def _positive_int(value: Any) -> Optional[int]:
return _positive(value, int)
def _positive_float(value: Any) -> Optional[float]:
return _positive(value, float)
return parsed if parsed is not None and parsed > 0 else None
def _cgroup_limit_bytes() -> Optional[int]:
"""Return the memory limit this process runs under, if cgroup-capped.
"""Memory limit this process runs under, if cgroup-capped.
Prefers cgroup v2 ``memory.high`` (the throttling point) over ``memory.max``,
then cgroup v1. Checks the process's *own* cgroup first (where a systemd
unit's ``MemoryHigh=``/``MemoryMax=`` lands — the root files read ``max``
there), then the root for container-style limits. ``max`` and the v1
near-2^63 sentinel mean unlimited.
Prefers v2 ``memory.high`` (the throttling point) over ``memory.max``, then v1.
Own cgroup first (where a systemd unit's ``MemoryHigh=``/``MemoryMax=`` lands —
root reads ``max`` there), then root for container-style limits. ``max`` and
the v1 near-2^63 sentinel mean unlimited.
"""
if sys.platform != "linux":
return None
candidates: list[str] = []
try:
from gateway.cgroup_cleanup import _own_cgroup_path
own = _own_cgroup_path()
except Exception:
own = None
if own and own != "/":
candidates += [f"/sys/fs/cgroup{own}/memory.high", f"/sys/fs/cgroup{own}/memory.max"]
candidates += [
"/sys/fs/cgroup/memory.high",
"/sys/fs/cgroup/memory.max",
"/sys/fs/cgroup/memory/memory.limit_in_bytes",
]
for candidate in candidates:
roots = ([f"/sys/fs/cgroup{own}"] if own and own != "/" else []) + ["/sys/fs/cgroup"]
for candidate in [f"{r}/memory.{f}" for r in roots for f in ("high", "max")] + ["/sys/fs/cgroup/memory/memory.limit_in_bytes"]:
try:
raw = Path(candidate).read_text(encoding="utf-8").strip()
except OSError:
limit = int(Path(candidate).read_text(encoding="utf-8").strip())
except (OSError, ValueError): # unreadable, empty, or "max"
continue
if not raw or raw == "max":
continue
try:
limit = int(raw)
except ValueError:
continue
if limit <= 0 or limit >= (1 << 62):
continue
return limit
if 0 < limit < (1 << 62):
return limit
return None
@@ -126,25 +95,16 @@ def _total_memory_bytes() -> Optional[int]:
def resolve_memory_high_mb(setting: Any) -> Optional[int]:
"""Resolve ``memory_high_mb`` into an absolute MB budget.
``"auto"`` derives it from the cgroup limit (or total RAM when uncapped);
a positive number is literal; anything falsy/off disables the pass.
"""
"""Absolute MB budget: ``"auto"`` derives from the cgroup limit (or total RAM when
uncapped); a positive number is literal; anything falsy/off disables the pass."""
if isinstance(setting, str):
normalized = setting.strip().lower()
if normalized != "auto":
return (
None
if normalized in ("", "off", "none", "false", "disabled")
else _positive_int(normalized)
)
elif isinstance(setting, bool):
if not setting:
return None
else:
return _positive_int(setting)
return None if normalized in _OFF_WORDS else _positive(normalized)
elif setting is False:
return None
elif setting is not True:
return _positive(setting)
limit = _cgroup_limit_bytes() or _total_memory_bytes()
if not limit:
return None
@@ -153,53 +113,29 @@ def resolve_memory_high_mb(setting: Any) -> Optional[int]:
def resolve_agent_cache_bounds(config: Any) -> AgentCacheBounds:
"""Read ``agent.agent_cache`` out of the *raw* config mapping.
The gateway's loader does not deep-merge ``DEFAULT_CONFIG``, so an absent
key stays absent and callers can tell "operator chose 128" from "unset".
"""
section: Any = None
if isinstance(config, dict):
agent_cfg = config.get("agent")
if isinstance(agent_cfg, dict):
section = agent_cfg.get("agent_cache")
"""Read ``agent.agent_cache`` from the *raw* config: the gateway loader does not
deep-merge ``DEFAULT_CONFIG``, so callers can tell "operator chose 128" from "unset"."""
section = (config.get("agent") or {}).get("agent_cache") if isinstance(config, dict) else None
if not isinstance(section, dict):
section = {}
max_evictions = _positive_int(section.get("max_evictions_per_pass"))
protect_recent = section.get("protect_recent")
protect_parsed = _positive_int(protect_recent)
# 0 means "shed anything" — distinct from unset. The bool guard keeps
# `protect_recent: false` (False == 0) on the default instead of silently
# disabling MRU protection.
if (
protect_parsed is None
and isinstance(protect_recent, int)
and not isinstance(protect_recent, bool)
and protect_recent == 0
):
protect_parsed = _positive(protect_recent)
# 0 means "shed anything" — distinct from unset. The bool guard keeps `protect_recent: false`
# (False == 0) on the default instead of silently disabling MRU protection.
if protect_parsed is None and _is_int(protect_recent) and protect_recent == 0:
protect_parsed = 0
return AgentCacheBounds(
max_size=_positive_int(section.get("max_size")),
idle_ttl_secs=_positive_float(section.get("idle_ttl_secs")),
max_size=_positive(section.get("max_size")),
idle_ttl_secs=_positive(section.get("idle_ttl_secs"), float),
memory_high_mb=resolve_memory_high_mb(section.get("memory_high_mb", "auto")),
max_evictions_per_pass=(
max_evictions if max_evictions is not None else _DEFAULT_MAX_EVICTIONS_PER_PASS
),
protect_recent=(
protect_parsed if protect_parsed is not None else _DEFAULT_PROTECT_RECENT
),
max_evictions_per_pass=_positive(section.get("max_evictions_per_pass")) or _DEFAULT_MAX_EVICTIONS_PER_PASS,
protect_recent=_DEFAULT_PROTECT_RECENT if protect_parsed is None else protect_parsed,
)
def read_anon_rss_mb() -> Optional[int]:
"""Return the process's anonymous resident memory in MB, or None.
Anonymous pages are where cached transcripts live; file-backed pages are
noise. ``collect_memory_snapshot`` reads ``/proc/self/status`` without a
dependency; psutil covers other platforms (total RSS only).
"""
"""Anonymous RSS in MB (where cached transcripts live; file-backed pages are noise),
or None. ``/proc/self/status`` first; psutil covers other platforms (total RSS only)."""
try:
from hermes_cli.mem_trim import collect_memory_snapshot
@@ -210,7 +146,6 @@ def read_anon_rss_mb() -> Optional[int]:
return kib // 1024
except Exception:
pass
try:
import psutil # type: ignore
@@ -220,37 +155,27 @@ def read_anon_rss_mb() -> Optional[int]:
def transcript_persistence_caught_up(agent: Any) -> bool:
"""True when the agent's live transcript is fully on disk.
"""True when the live transcript is fully on disk.
Soft eviction drops ``_session_messages`` and rebuilds from the persisted
session, so it is only safe once ``_last_flushed_db_idx`` (advanced to
``len(messages)`` by ``AIAgent._flush_messages_to_session_db`` only on a
fully successful write) has caught up. Unknown shapes are *not* caught up:
a skipped eviction costs memory, a wrong one costs the conversation.
Soft eviction rebuilds from the persisted session, so it is only safe once
``_last_flushed_db_idx`` (advanced only on a fully successful write) has caught
up. Unknown shapes are *not* caught up: a skipped eviction costs memory, a
wrong one costs the conversation.
"""
messages = getattr(agent, "_session_messages", None)
if not isinstance(messages, list):
return False
flushed = getattr(agent, "_last_flushed_db_idx", None)
if not isinstance(flushed, int) or isinstance(flushed, bool):
return False
return flushed >= len(messages)
messages, flushed = getattr(agent, "_session_messages", None), getattr(agent, "_last_flushed_db_idx", None)
return isinstance(messages, list) and _is_int(flushed) and flushed >= len(messages)
def plan_pressure_evictions(
ordered_entries: Iterable[Tuple[str, Any]],
*,
is_evictable: Callable[[str, Any], bool],
max_evictions: int,
protect_recent: int = 0,
ordered_entries: Iterable[Tuple[str, Any]], *, is_evictable: Callable[[str, Any], bool],
max_evictions: int, protect_recent: int = 0,
) -> List[Tuple[str, Any]]:
"""Choose which cached sessions to shed, least-recently-used first.
``ordered_entries`` must be LRU→MRU (the cache OrderedDict is kept that way
by ``move_to_end`` on every hit). The batch is capped so one pass cannot
stall the gateway. ``protect_recent`` is clamped to half the cache: a few
huge transcripts can exhaust the budget alone, and a fixed guard would then
protect the whole cache with nothing left to shed.
``ordered_entries`` must be LRU→MRU (the cache OrderedDict ``move_to_end``s on
every hit). The batch is capped so one pass cannot stall the gateway.
``protect_recent`` is clamped to half the cache: a few huge transcripts can
exhaust the budget alone, and a fixed guard would leave nothing to shed.
"""
entries = list(ordered_entries)
if max_evictions <= 0 or not entries:
@@ -258,7 +183,6 @@ def plan_pressure_evictions(
protect = min(max(protect_recent, 0), len(entries) // 2)
if protect:
entries = entries[:-protect]
plan: List[Tuple[str, Any]] = []
for key, agent in entries:
if len(plan) >= max_evictions:
gateway/authz_mixin.py
+243 -384
View File
@@ -1,15 +1,13 @@
"""User-authorization methods for ``GatewayRunner``.
Inbound-message authorization cluster: whether a user/chat may talk to the
"""User-authorization mixin for ``GatewayRunner``: may this user/chat talk to the
agent, the per-adapter DM policy, and the unauthorized-DM behavior.
``self.*`` calls resolve via the MRO. ``gateway.run`` is never imported at
module import time (cycle); the one method that logs imports its ``logger``
lazily so records keep the ``"gateway.run"`` logger name.
``gateway.run`` is never imported at module import time (cycle); the one method
that logs imports its ``logger`` lazily so records keep the ``"gateway.run"`` name.
"""
from __future__ import annotations
import contextlib
import os
from typing import Optional
@@ -22,6 +20,7 @@ from gateway.whatsapp_identity import (
)
_GROUP_CHAT_TYPES = frozenset({"group", "forum", "channel"})
_GROUP_FORUM_TYPES = frozenset({"group", "forum"})
_TRUTHY = frozenset({"true", "1", "yes"})
# Platform -> ``<PLATFORM>_ALLOWED_USERS`` / ``<PLATFORM>_ALLOW_ALL_USERS``.
@@ -47,51 +46,39 @@ _ALLOW_BOTS_ENV = {
def _platform_gate_env(name: str, default: str = "") -> str:
"""Read an allow/deny gate env var with per-profile isolation.
When a profile secret scope is installed AND multiplexing is active, a
scoped miss returns ``default`` instead of falling through to
``os.environ``: the process env may hold ANOTHER profile's first-writer
bridged value, so a fallthrough would leak allowlists across profiles.
With a profile secret scope installed AND multiplexing active, a scoped miss
returns ``default`` instead of falling through to ``os.environ``, which may
hold ANOTHER profile's first-writer bridged value (allowlist leak).
Single-profile deployments behave exactly like ``os.getenv``.
"""
if not name:
return default
try:
with contextlib.suppress(Exception):
from agent.secret_scope import current_secret_scope, is_multiplex_active
scope = current_secret_scope()
if scope is not None and is_multiplex_active():
val = scope.get(name)
return default if val is None else str(val).strip()
except Exception:
pass
return (os.getenv(name) or default).strip()
_auth_env = _platform_gate_env
def _env_truthy(name: str) -> bool:
return _auth_env(name).lower() in _TRUTHY
def _registry_entry(platform):
"""Platform-registry entry for a (plugin) platform, or None."""
if platform is None:
return None
try:
with contextlib.suppress(Exception):
from gateway.platform_registry import platform_registry
return platform_registry.get(platform.value)
except Exception:
return None
def _platform_declares_allowed_users_env(platform) -> bool:
"""Whether a plugin platform's registry entry declares ``allowed_users_env``.
Such platforms (Buzz, DingTalk, ...) document ``PlatformConfig.extra
.allowed_users`` as the config-file spelling of that env allowlist, so the
live adapter's extra is a valid authorization source when the env var is
absent. Built-in platforms and unknown entries return False.
"""
entry = _registry_entry(platform)
return bool(entry and entry.allowed_users_env)
return None
def _coerce_allow_set(raw) -> set[str]:
@@ -107,11 +94,12 @@ def _allows(allowed: set[str], candidate: Optional[str]) -> bool:
return "*" in allowed or candidate in allowed
# ---------------------------------------------------------------------------
# Nostr npub -> hex (Buzz). ``BUZZ_ALLOWED_USERS`` accepts hex or ``npub1…``
# but inbound pubkeys are always hex, so npub entries must be decoded.
# Pure stdlib; mirrors plugins/platforms/buzz/adapter.py.
# ---------------------------------------------------------------------------
def _adapter_config_extra(adapter) -> dict:
return getattr(getattr(adapter, "config", None), "extra", None) or {}
# Nostr npub -> hex (Buzz): ``BUZZ_ALLOWED_USERS`` accepts hex or ``npub1…`` but
# inbound pubkeys are always hex. Pure stdlib; mirrors plugins/platforms/buzz/adapter.py.
_BECH32_CHARSET = "qpzry9x8gf2tvdw0s3jn54khce6mua7l"
@@ -180,25 +168,50 @@ def _normalize_nostr_allow_entries(entries: set) -> set:
return expanded
def _principal_matches_allowlist(source, user_id: str, allowed_ids: set) -> bool:
"""Whether *user_id* (under any platform-specific alias) is in *allowed_ids*."""
check_ids = {user_id}
if "@" in user_id:
check_ids.add(user_id.split("@")[0])
# WhatsApp (Baileys + Cloud): phone<->LID / JID aliases match the same principal.
if source.platform in {Platform.WHATSAPP, Platform.WHATSAPP_CLOUD}:
allowed_ids = set().union(*(_expand_whatsapp_auth_aliases(a) for a in allowed_ids)) or allowed_ids
check_ids.update(_expand_whatsapp_auth_aliases(user_id))
normalized_user_id = _normalize_whatsapp_identifier(user_id)
if normalized_user_id:
check_ids.add(normalized_user_id)
platform_value = source.platform.value if source.platform is not None else None
# SimpleX: user_id is the numeric contactId but the UI only shows display names.
if platform_value == "simplex" and source.user_name:
check_ids.add(source.user_name)
# Buzz: allowlist may hold npub or hex; inbound pubkeys are hex.
if platform_value == "buzz":
allowed_ids = _normalize_nostr_allow_entries(allowed_ids)
hex_user = _npub_to_hex(user_id) if user_id.startswith("npub") else None
if hex_user:
check_ids.add(hex_user)
return bool(check_ids & allowed_ids)
class GatewayAuthorizationMixin:
"""User/chat authorization methods for ``GatewayRunner``."""
# ``getattr(self, ...)`` throughout: test helpers build bare runners via
# ``object.__new__`` without ``adapters`` / ``config`` (pitfalls.md #17).
# ``object.__new__`` without ``adapters`` / ``config``.
def _authorization_adapter(
self,
platform: Optional[Platform],
profile: Optional[str] = None,
):
"""Resolve the live adapter whose intake policy should gate authorization.
def _primary_adapters(self) -> dict:
return getattr(self, "adapters", None) or {}
Secondary-profile adapters live in ``_profile_adapters[profile]``; the
launched (primary) profile owns ``self.adapters``. ``_profile_adapters``
is consulted BEFORE comparing against the active profile name: multiplex
turns override ``HERMES_HOME`` so ``_active_profile_name()`` reports the
secondary profile mid-turn, and treating it as primary would hand it the
default bot (or default-deny secondary-only platforms like A2A).
def _authorization_adapter(self, platform: Optional[Platform], profile: Optional[str] = None):
"""Live adapter whose intake policy gates authorization.
Secondary-profile adapters live in ``_profile_adapters[profile]``; the primary
profile owns ``self.adapters``. ``_profile_adapters`` is consulted BEFORE the
active profile name: multiplex turns override ``HERMES_HOME`` so
``_active_profile_name()`` reports the secondary profile mid-turn, and treating
it as primary would hand it the default bot.
"""
if not platform:
return None
@@ -207,23 +220,17 @@ class GatewayAuthorizationMixin:
profile_adapters = getattr(self, "_profile_adapters", None) or {}
if profile_name in profile_adapters:
return profile_adapters[profile_name].get(platform)
# Compare against the identity captured at construction, not the
# per-turn HERMES_HOME-derived name.
# Identity captured at construction, not the per-turn HERMES_HOME-derived name.
primary_profile = getattr(self, "_primary_profile_name", None)
if not primary_profile:
active_profile_fn = getattr(self, "_active_profile_name", None)
if callable(active_profile_fn):
try:
primary_profile = active_profile_fn()
except Exception:
primary_profile = None
with contextlib.suppress(Exception):
primary_profile = self._active_profile_name()
if profile_name == primary_profile:
return (getattr(self, "adapters", None) or {}).get(platform)
# Fail closed: a stamped secondary profile with no registry entry
# (adapter failed to connect) must NOT fall back to the default
# profile's adapter -- that sends replies out the wrong bot.
return self._primary_adapters().get(platform)
# Fail closed: a secondary profile whose adapter failed to connect must NOT
# fall back to the default profile's adapter (replies out the wrong bot).
return None
return (getattr(self, "adapters", None) or {}).get(platform)
return self._primary_adapters().get(platform)
def _adapter_for_source(self, source: Optional[SessionSource]):
"""Resolve the live adapter for an inbound ``SessionSource``."""
@@ -232,22 +239,17 @@ class GatewayAuthorizationMixin:
transport_adapter = self._registered_transport_adapter(source)
if transport_adapter is not None:
return transport_adapter
# Relay ingress keeps the underlying platform on the source (session
# keys / display policy), but delivery must use the one process-level
# RelayAdapter that owns the connector socket -- secondary profiles do
# not register their own, so a profile-aware lookup would silently
# disable streaming/typing/tool progress.
# Relay ingress keeps the underlying platform on the source, but delivery must
# use the one process-level RelayAdapter owning the connector socket; a
# profile-aware lookup would silently disable streaming/typing/tool progress.
if getattr(source, "delivered_via_upstream_relay", False) is True:
return (getattr(self, "adapters", None) or {}).get(Platform.RELAY)
return self._primary_adapters().get(Platform.RELAY)
# ``getattr``: test fixtures build bare SimpleNamespace sources without ``profile``.
return self._authorization_adapter(
getattr(source, "platform", None),
getattr(source, "profile", None),
)
return self._authorization_adapter(getattr(source, "platform", None), getattr(source, "profile", None))
def _owning_profile(self, adapter, platform):
"""Return (registered, profile) for a live adapter: profile is None for primary."""
if adapter is (getattr(self, "adapters", None) or {}).get(platform):
if adapter is self._primary_adapters().get(platform):
return True, None
for profile, profile_adapters in (getattr(self, "_profile_adapters", None) or {}).items():
if adapter is profile_adapters.get(platform):
@@ -255,13 +257,12 @@ class GatewayAuthorizationMixin:
return False, None
def _registered_transport_adapter(self, source: SessionSource):
"""Return the registered adapter that created *source*, if retained.
"""The registered adapter that created *source*, if retained.
``source.profile`` is the runtime/session namespace and may differ from
the adapter profile when one shared credential serves several routed
runtimes; ``build_source`` keeps the receiving adapter as in-process
provenance so replies and intake-policy checks stay on that transport.
Restored or hand-built sources fall back (fail-closed) to profile lookup.
``source.profile`` may differ from the adapter profile when one shared credential
serves several routed runtimes; ``build_source`` keeps the receiving adapter as
provenance so replies stay on that transport. Restored/hand-built sources fall
back (fail-closed) to profile lookup.
"""
adapter_ref = getattr(source, "_transport_adapter_ref", None)
adapter = adapter_ref() if callable(adapter_ref) else None
@@ -286,102 +287,57 @@ class GatewayAuthorizationMixin:
adapter = self._authorization_adapter(platform, profile)
return adapter is not None and bool(getattr(adapter, name, False))
def _adapter_authorization_is_upstream(
self,
platform: Optional[Platform],
*,
profile: Optional[str] = None,
) -> bool:
"""Whether the adapter delegates authz to a trusted authenticated upstream.
def _adapter_authorization_is_upstream(self, platform: Optional[Platform], *, profile: Optional[str] = None) -> bool:
"""Whether the adapter delegates authz to a trusted authenticated upstream (relay).
Mirrors ``BasePlatformAdapter.authorization_is_upstream`` (True for the
relay adapter: the connector authenticates the WS and resolves owner
bindings before delivering). Unlike ``_adapter_enforces_own_access_policy``
(a LOCAL policy only trusted when it is an allowlist) this UPSTREAM
decision is honored directly. False when the adapter is unknown.
Unlike ``_adapter_enforces_own_access_policy`` (a LOCAL policy only trusted when
it is an allowlist) this UPSTREAM decision is honored directly. False when unknown.
"""
return self._adapter_flag(platform, "authorization_is_upstream", profile)
def _adapter_enforces_own_access_policy(
self,
platform: Optional[Platform],
*,
profile: Optional[str] = None,
) -> bool:
"""Whether the adapter gates access at intake itself.
def _adapter_enforces_own_access_policy(self, platform: Optional[Platform], *, profile: Optional[str] = None) -> bool:
"""Whether the adapter gates access at intake itself (WeCom, Weixin, Yuanbao, QQBot, WhatsApp).
Mirrors ``BasePlatformAdapter.enforces_own_access_policy`` (WeCom, Weixin,
Yuanbao, QQBot, WhatsApp). The flag alone is NOT "already authorized":
these adapters default to ``open``, so ``_is_user_authorized`` only trusts
them when the effective policy is an actual ``allowlist``.
The flag alone is NOT "already authorized": these adapters default to ``open``,
so ``_is_user_authorized`` only trusts them under an actual ``allowlist`` policy.
"""
return self._adapter_flag(platform, "enforces_own_access_policy", profile)
def _config_extra(self, platform) -> dict:
"""``config.platforms[platform].extra`` as a dict ({} when absent)."""
config = getattr(self, "config", None)
platform_cfg = (
config.platforms.get(platform)
if config is not None and hasattr(config, "platforms")
else None
)
extra = getattr(platform_cfg, "extra", None) if platform_cfg else None
platforms = getattr(getattr(self, "config", None), "platforms", None)
extra = getattr(platforms.get(platform), "extra", None) if platforms is not None else None
return extra if isinstance(extra, dict) else {}
def _adapter_setting(self, platform, attr: str, extra_key: str, profile):
"""Live adapter's resolved ``attr``, else ``config.extra[extra_key]``.
The adapter value is preferred because it already folds in the
``<PLATFORM>_*`` env override, which is not always bridged back into
``config.extra``; the extra fallback serves bare runners with no adapter.
"""
"""Live adapter's resolved ``attr`` (folds in the ``<PLATFORM>_*`` env override),
else ``config.extra[extra_key]`` for bare runners with no adapter."""
adapter = self._authorization_adapter(platform, profile)
value = getattr(adapter, attr, None) if adapter is not None else None
if value is None:
value = self._config_extra(platform).get(extra_key)
return value
def _adapter_dm_policy(
self,
platform: Optional[Platform],
*,
profile: Optional[str] = None,
) -> str:
"""Lowercased effective ``dm_policy`` (open/allowlist/disabled/pairing), ``""`` if unknown.
"Reached the gateway" only carries an authorization signal in the
``allowlist`` case; ``open`` forwards everyone and ``pairing`` forwards
unpaired DMs for the handshake.
"""
def _adapter_policy(self, platform, attr: str, extra_key: str, profile) -> str:
if not platform:
return ""
policy = self._adapter_setting(platform, "_dm_policy", "dm_policy", profile)
return str(policy or "").strip().lower()
return str(self._adapter_setting(platform, attr, extra_key, profile) or "").strip().lower()
def _adapter_group_policy(
self,
platform: Optional[Platform],
*,
profile: Optional[str] = None,
) -> str:
def _adapter_dm_policy(self, platform: Optional[Platform], *, profile: Optional[str] = None) -> str:
"""Lowercased effective ``dm_policy`` (open/allowlist/disabled/pairing), ``""`` if unknown."""
return self._adapter_policy(platform, "_dm_policy", "dm_policy", profile)
def _adapter_group_policy(self, platform: Optional[Platform], *, profile: Optional[str] = None) -> str:
"""Lowercased effective ``group_policy`` (open/allowlist/disabled), ``""`` if unknown."""
if not platform:
return ""
policy = self._adapter_setting(platform, "_group_policy", "group_policy", profile)
return str(policy or "").strip().lower()
return self._adapter_policy(platform, "_group_policy", "group_policy", profile)
def _adapter_group_has_sender_allowlist(
self,
platform: Optional[Platform],
chat_id: Optional[str],
*,
profile: Optional[str] = None,
self, platform: Optional[Platform], chat_id: Optional[str], *, profile: Optional[str] = None
) -> bool:
"""Whether a per-group sender allowlist (WeCom ``groups.<id>.allow_from``) gated this message.
A group may be open at the chat level while restricting senders; if such
a message reached the gateway the adapter already checked that sender
allowlist, so it is a trustworthy intake decision.
A group may be open at the chat level while restricting senders; reaching the
gateway then means the adapter already checked that list.
"""
if not platform or not chat_id:
return False
@@ -393,12 +349,10 @@ class GatewayAuthorizationMixin:
group_cfg = groups.get(chat_id_str)
if not isinstance(group_cfg, dict):
lowered = chat_id_str.lower()
for key, value in groups.items():
if isinstance(key, str) and key.lower() == lowered and isinstance(value, dict):
group_cfg = value
break
if not isinstance(group_cfg, dict):
group_cfg = groups.get("*")
group_cfg = next(
(v for k, v in groups.items() if isinstance(k, str) and k.lower() == lowered and isinstance(v, dict)),
groups.get("*"),
)
if not isinstance(group_cfg, dict):
return False
@@ -418,43 +372,100 @@ class GatewayAuthorizationMixin:
return getattr(self, "pairing_store", None)
def _adapter_extra_for_source(self, source) -> dict:
return _adapter_config_extra(self._adapter_for_source(source))
def _own_policy_authorizes(self, source, user_id, is_group, adapter_profile) -> Optional[bool]:
"""Own-policy adapter verdict when no env allowlist exists; None = no verdict.
Trusted only when the effective policy for THIS chat type is ``allowlist``:
``open`` forwards EVERY sender (the fail-open SECURITY.md §2.6 forbids),
``disabled`` never forwards, ``pairing`` forwards unpaired DMs for the handshake
(already denied by the pairing-store check). Anything else → default-deny.
"""
if is_group:
if self._adapter_group_has_sender_allowlist(source.platform, source.chat_id, profile=adapter_profile):
return True
effective_policy = self._adapter_group_policy(source.platform, profile=adapter_profile)
else:
effective_policy = self._adapter_dm_policy(source.platform, profile=adapter_profile)
if effective_policy != "allowlist":
return None
# Re-check DMs via the live adapter's ``_is_dm_allowed`` when present: pairing
# revoke can clear WHATSAPP_ALLOWED_USERS while a construction-time snapshot
# would keep authorizing until restart. Others keep the historical rubber-stamp.
if not is_group:
adapter = self._authorization_adapter(source.platform, profile=adapter_profile)
dm_check = getattr(adapter, "_is_dm_allowed", None) if adapter is not None else None
if callable(dm_check):
return bool(dm_check(user_id))
return True
def _adapter_extra_allowlist_authorizes(self, source, user_id, is_group) -> bool:
"""Adapters (e.g. Telegram) that gate via config.extra.allow_from / group_allow_from
without setting enforces_own_access_policy."""
adapter = self._adapter_for_source(source)
if adapter is None:
return {}
return getattr(getattr(adapter, "config", None), "extra", None) or {}
return False
extra = _adapter_config_extra(adapter)
adapter_allow = extra.get("group_allow_from" if is_group else "allow_from")
if not adapter_allow:
# Plugin platforms (Buzz, DingTalk) spell their env allowlist as
# ``extra.allowed_users``; under multiplex only the default profile's list
# reaches the env (first-writer-wins bridge), so read the live adapter's.
entry = _registry_entry(source.platform)
if entry and entry.allowed_users_env:
adapter_allow = extra.get("allowed_users")
if not adapter_allow:
return False
allowed = _coerce_allow_set(adapter_allow)
normalize = getattr(adapter, "normalize_user_id", None)
if callable(normalize):
# Ids and entries may spell the same principal differently (Buzz hex vs npub).
allowed = {normalize(entry) or entry for entry in allowed}
return _allows(allowed, user_id)
def _is_user_authorized(
self,
source: SessionSource,
*,
allow_adapter_delegation: bool = True,
) -> bool:
"""Check if a user is authorized to use the bot.
def _adapter_resolved_allowlist_ids(self, source) -> set[str]:
"""IDs an adapter resolved from username-shaped allowlist entries at connect time (Discord).
The per-turn .env hot-reload restores RAW usernames, so from the second turn on the
env allowlist holds usernames while user_id is numeric. Never a widening: the
empty-allowlist branch already returned and adapters only resolve operator-written
entries. Only called with a non-empty platform allowlist so group/global-only
configs never consult adapter memory; type-checked so mocks cannot auto-truthy in.
"""
adapter = resolved_ids = None
with contextlib.suppress(Exception):
adapter = self._adapter_for_source(source)
resolver = getattr(adapter, "resolved_allowlist_user_ids", None)
if callable(resolver):
with contextlib.suppress(Exception):
resolved_ids = resolver()
if not isinstance(resolved_ids, (set, frozenset, list, tuple)):
return set()
return {str(entry).strip() for entry in resolved_ids if isinstance(entry, (str, int)) and str(entry).strip()}
def _is_user_authorized(self, source: SessionSource, *, allow_adapter_delegation: bool = True) -> bool:
"""Whether a user may use the bot.
Order: trusted-upstream delegation, chat-scoped group allowlists,
``{PLATFORM}_ALLOW_BOTS``, per-platform allow-all, adapter role auth,
pairing store, env/config allowlists, ``GATEWAY_ALLOW_ALL_USERS``,
default deny.
``{PLATFORM}_ALLOW_BOTS``, per-platform allow-all, adapter role auth, pairing
store, env/config allowlists, ``GATEWAY_ALLOW_ALL_USERS``, default deny.
"""
from gateway.run import logger
# HA events are system-generated (HASS_TOKEN authenticates the
# connection); webhook events are HMAC-verified in the adapter.
# HA events are system-generated (HASS_TOKEN); webhook events are HMAC-verified.
if source.platform in {Platform.HOMEASSISTANT, Platform.WEBHOOK}:
return True
adapter_profile = self._adapter_profile_for_source(source)
is_group = source.chat_type in _GROUP_CHAT_TYPES
is_group_or_forum = source.chat_type in _GROUP_FORUM_TYPES
# Trusted-upstream delegation (relay): the Team Gateway connector
# authenticates this gateway's WS and resolves owner-only author
# bindings BEFORE delivering, so the event is already authorized and
# there is no local RELAY_ALLOWED_USERS to consult. Not a fail-open: it
# fires only for events actually delivered over the authenticated relay
# WS (transport stamps ``delivered_via_upstream_relay``) or whose
# adapter declares ``authorization_is_upstream=True``. The delivery
# marker is PRIMARY because a relayed message carries the UNDERLYING
# platform (discord/...), not ``Platform.RELAY``; the adapter-flag check
# covers events whose platform IS RELAY (interaction passthrough).
# Trusted-upstream delegation (relay): the connector authenticates this
# gateway's WS and resolves owner bindings BEFORE delivering, so there is no
# local RELAY_ALLOWED_USERS. Not a fail-open: fires only for events actually
# delivered over the relay WS (``delivered_via_upstream_relay``) or whose adapter
# declares ``authorization_is_upstream``. The delivery marker is PRIMARY because
# a relayed message carries the UNDERLYING platform, not ``Platform.RELAY``.
# ``is True``: a MagicMock stand-in must not auto-truthy into authz.
if allow_adapter_delegation and (
source.delivered_via_upstream_relay is True
@@ -464,33 +475,21 @@ class GatewayAuthorizationMixin:
user_id = source.user_id
# Chat-scoped group allowlists (TELEGRAM_GROUP_ALLOWED_CHATS /
# QQ_GROUP_ALLOWED_USERS) must work with ``user_id is None`` (anonymous
# admins, sender_chat posts, channel broadcasts), so they run before
# the no-user-id guard.
# Chat-scoped group allowlists must work with ``user_id is None`` (anonymous
# admins, sender_chat posts, channel broadcasts): run before the no-user-id guard.
if is_group and source.chat_id:
chat_allowlist_env = _GROUP_CHAT_ENV.get(source.platform, "")
if chat_allowlist_env and _allows(
_coerce_allow_set(_platform_gate_env(chat_allowlist_env)), source.chat_id
):
if chat_allowlist_env and _allows(_coerce_allow_set(_platform_gate_env(chat_allowlist_env)), source.chat_id):
return True
# config.yaml fallback (``extra.group_allowed_chats``): Telegram
# observe-unmentioned mode strips user_id from triggered group
# messages, so the env-only check above misses config allowlists.
try:
adapter_group_allowed = self._adapter_extra_for_source(source).get(
"group_allowed_chats"
)
if adapter_group_allowed and _allows(
_coerce_allow_set(adapter_group_allowed), source.chat_id
):
# config.yaml fallback (``extra.group_allowed_chats``): Telegram observe-
# unmentioned mode strips user_id, so the env-only check above misses it.
with contextlib.suppress(Exception):
adapter_group_allowed = self._adapter_extra_for_source(source).get("group_allowed_chats")
if adapter_group_allowed and _allows(_coerce_allow_set(adapter_group_allowed), source.chat_id):
return True
except Exception:
pass
# Bots admitted by {PLATFORM}_ALLOW_BOTS bypass the human allowlist.
# Also before the no-user-id guard: Slack Workflow Builder posts arrive
# as bot_message with user=None.
# Bots admitted by {PLATFORM}_ALLOW_BOTS bypass the human allowlist. Also before
# the no-user-id guard: Slack Workflow Builder posts arrive with user=None.
if getattr(source, "is_bot", False):
allow_bots_var = _ALLOW_BOTS_ENV.get(source.platform)
if allow_bots_var and _platform_gate_env(allow_bots_var, "none").lower().strip() in {"mentions", "all"}:
@@ -499,120 +498,60 @@ class GatewayAuthorizationMixin:
if not user_id:
return False
platform_env_map = dict(_ALLOWED_USERS_ENV)
platform_allow_all_map = dict(_ALLOW_ALL_ENV)
if source.platform not in platform_env_map:
try:
entry = _registry_entry(source.platform)
if entry:
if entry.allowed_users_env:
platform_env_map[source.platform] = entry.allowed_users_env
if entry.allow_all_env:
platform_allow_all_map[source.platform] = entry.allow_all_env
except Exception:
pass
platform_allow_env = _ALLOWED_USERS_ENV.get(source.platform, "")
platform_allow_all_var = _ALLOW_ALL_ENV.get(source.platform, "")
if source.platform not in _ALLOWED_USERS_ENV:
entry = _registry_entry(source.platform)
with contextlib.suppress(Exception):
platform_allow_env = getattr(entry, "allowed_users_env", "") or platform_allow_env
platform_allow_all_var = getattr(entry, "allow_all_env", "") or platform_allow_all_var
platform_allow_all_var = platform_allow_all_map.get(source.platform, "")
if platform_allow_all_var and _auth_env(platform_allow_all_var).lower() in _TRUTHY:
if platform_allow_all_var and _env_truthy(platform_allow_all_var):
return True
# Adapter-verified role auth (Discord confirmed DISCORD_ALLOWED_ROLES
# before dispatch). ``is True`` so a MagicMock source cannot pass.
# Adapter-verified role auth (Discord DISCORD_ALLOWED_ROLES). ``is True``: no MagicMock pass.
if allow_adapter_delegation and getattr(source, "role_authorized", False) is True:
return True
# Pairing store: a first-class grant created only by a trusted operator
# approving a code (inbound senders can never reach approve_code).
# Honored as a UNION with the allowlist; approval also mirrors the user
# into a configured allowlist (PairingStore._approve_user).
# Pairing store: a first-class grant created only by an operator approving a
# code. Honored as a UNION with the allowlist (approval also mirrors into it).
platform_name = source.platform.value if source.platform else ""
pairing_store = self._pairing_store_for(source)
if pairing_store is not None and pairing_store.is_approved(platform_name, user_id):
return True
platform_allowlist = _auth_env(platform_env_map.get(source.platform, ""))
platform_allowlist = _auth_env(platform_allow_env)
group_user_allowlist = ""
group_chat_allowlist = ""
if source.chat_type in {"group", "forum"}:
if is_group_or_forum:
group_user_allowlist = _auth_env(_GROUP_USER_ENV.get(source.platform, ""))
group_chat_allowlist = _auth_env(_GROUP_CHAT_ENV.get(source.platform, ""))
global_allowlist = _auth_env("GATEWAY_ALLOWED_USERS")
if not platform_allowlist and not group_user_allowlist and not group_chat_allowlist and not global_allowlist:
# No env allowlist. Own-policy adapters gate at intake, but their
# decision is only trusted when the effective policy for THIS chat
# type is ``allowlist``: ``open`` forwards EVERY sender (trusting it
# is the fail-open SECURITY.md §2.6 forbids), ``disabled`` never
# forwards, ``pairing`` forwards unpaired DMs for the handshake
# (already denied by the pairing-store check above). Anything else
# falls through to default-deny; GATEWAY_ALLOW_ALL_USERS, the
# per-platform ALLOW_ALL flag and pairing stay the explicit opt-ins.
if not (platform_allowlist or group_user_allowlist or group_chat_allowlist or global_allowlist):
# No env allowlist: own-policy adapters gate at intake (see _own_policy_authorizes).
if allow_adapter_delegation and self._adapter_enforces_own_access_policy(
source.platform, profile=adapter_profile
):
if is_group:
effective_policy = self._adapter_group_policy(source.platform, profile=adapter_profile)
if self._adapter_group_has_sender_allowlist(
source.platform, source.chat_id, profile=adapter_profile
):
return True
else:
effective_policy = self._adapter_dm_policy(source.platform, profile=adapter_profile)
if effective_policy == "allowlist":
# Re-check DMs against the live adapter when it exposes
# ``_is_dm_allowed``: pairing revoke can clear
# WHATSAPP_ALLOWED_USERS while a construction-time
# ``_allow_from`` snapshot would keep authorizing until
# restart. Adapters without the helper keep the historical
# "reached the gateway under allowlist policy" rubber-stamp.
if not is_group:
adapter = self._authorization_adapter(source.platform, profile=adapter_profile)
dm_check = getattr(adapter, "_is_dm_allowed", None) if adapter is not None else None
if callable(dm_check):
return bool(dm_check(user_id))
return True
# Adapters (e.g. Telegram) that gate via config.extra.allow_from /
# group_allow_from without setting enforces_own_access_policy.
adapter = self._adapter_for_source(source)
if adapter is not None:
extra = getattr(getattr(adapter, "config", None), "extra", None) or {}
adapter_allow = extra.get("group_allow_from" if is_group else "allow_from")
if not adapter_allow and _platform_declares_allowed_users_env(source.platform):
# Plugin platforms (Buzz) carry the env allowlist as
# ``extra.allowed_users``. Under multiplex the YAML->env
# bridge is first-writer-wins, so only the default profile's
# list reaches the env read above; consult the live
# profile-routed adapter's config instead.
adapter_allow = extra.get("allowed_users")
if adapter_allow:
allowed = _coerce_allow_set(adapter_allow)
normalize = getattr(adapter, "normalize_user_id", None)
if callable(normalize):
# Ids and entries may spell the same principal
# differently (Buzz hex vs npub).
allowed = {normalize(entry) or entry for entry in allowed}
if _allows(allowed, user_id):
return True
return _auth_env("GATEWAY_ALLOW_ALL_USERS").lower() in _TRUTHY
verdict = self._own_policy_authorizes(source, user_id, is_group, adapter_profile)
if verdict is not None:
return verdict
if self._adapter_extra_allowlist_authorizes(source, user_id, is_group):
return True
return _env_truthy("GATEWAY_ALLOW_ALL_USERS")
# Telegram group traffic authorized by chat ID (separate from
# TELEGRAM_GROUP_ALLOWED_USERS, which gates the sender).
if group_chat_allowlist and source.chat_type in {"group", "forum"} and source.chat_id:
if _allows(_coerce_allow_set(group_chat_allowlist), source.chat_id):
return True
# Backward-compat: TELEGRAM_GROUP_ALLOWED_USERS was once (mis)used as a
# chat-ID allowlist. "-"-prefixed values are chat IDs, so honor them as
# such and warn once; the correct var is TELEGRAM_GROUP_ALLOWED_CHATS.
if (
source.platform == Platform.TELEGRAM
and group_user_allowlist
and source.chat_type in {"group", "forum"}
and source.chat_id
group_chat_allowlist and is_group_or_forum and source.chat_id
and _allows(_coerce_allow_set(group_chat_allowlist), source.chat_id)
):
legacy_chat_ids = {
v.strip() for v in group_user_allowlist.split(",") if v.strip().startswith("-")
}
return True
# Backward-compat: TELEGRAM_GROUP_ALLOWED_USERS was once (mis)used as a chat-ID
# allowlist; "-"-prefixed values are chat IDs, honor them and warn once.
if source.platform == Platform.TELEGRAM and group_user_allowlist and is_group_or_forum and source.chat_id:
legacy_chat_ids = {v.strip() for v in group_user_allowlist.split(",") if v.strip().startswith("-")}
if legacy_chat_ids:
if not getattr(self, "_warned_telegram_group_users_legacy", False):
logger.warning(
@@ -626,113 +565,39 @@ class GatewayAuthorizationMixin:
if source.chat_id in legacy_chat_ids:
return True
# In group/forum chats TELEGRAM_GROUP_ALLOWED_USERS is the scoped
# allowlist and does not imply DM access; TELEGRAM_ALLOWED_USERS is
# platform-wide and works everywhere.
# TELEGRAM_GROUP_ALLOWED_USERS is group-scoped (no DM access); TELEGRAM_ALLOWED_USERS is platform-wide.
allowed_ids = (
_coerce_allow_set(platform_allowlist)
| _coerce_allow_set(group_user_allowlist)
| _coerce_allow_set(global_allowlist)
)
# Adapters that resolve username-shaped entries to numeric IDs at
# connect time (Discord) keep the resolved set in memory; the per-turn
# .env hot-reload restores the RAW usernames into the env, so from the
# second turn on ``platform_allowlist`` holds usernames while user_id is
# numeric. Union in the adapter's resolved IDs. Never a widening: the
# empty-allowlist branch already returned and adapters only resolve
# operator-written entries. Guarded on ``platform_allowlist`` so
# group/global-only configs never consult adapter memory; duck-typed +
# type-checked so mock adapters cannot auto-truthy in.
if platform_allowlist:
try:
adapter = self._adapter_for_source(source)
except Exception:
adapter = None
resolver = getattr(adapter, "resolved_allowlist_user_ids", None)
if callable(resolver):
try:
resolved_ids = resolver()
except Exception:
resolved_ids = None
if isinstance(resolved_ids, (set, frozenset, list, tuple)):
allowed_ids.update(
str(entry).strip()
for entry in resolved_ids
if isinstance(entry, (str, int)) and str(entry).strip()
)
allowed_ids |= self._adapter_resolved_allowlist_ids(source)
return "*" in allowed_ids or _principal_matches_allowlist(source, user_id, allowed_ids)
if "*" in allowed_ids:
return True
def _get_unauthorized_dm_behavior(self, platform: Optional[Platform], *, profile: Optional[str] = None) -> str:
"""How unauthorized DMs are handled ("pair" / "ignore") for a platform.
check_ids = {user_id}
if "@" in user_id:
check_ids.add(user_id.split("@")[0])
# WhatsApp (Baileys + Cloud): resolve phone<->LID / JID aliases so
# device-suffix and bare-phone entries match the same principal.
if source.platform in {Platform.WHATSAPP, Platform.WHATSAPP_CLOUD}:
normalized_allowed_ids = set()
for allowed_id in allowed_ids:
normalized_allowed_ids.update(_expand_whatsapp_auth_aliases(allowed_id))
if normalized_allowed_ids:
allowed_ids = normalized_allowed_ids
check_ids.update(_expand_whatsapp_auth_aliases(user_id))
normalized_user_id = _normalize_whatsapp_identifier(user_id)
if normalized_user_id:
check_ids.add(normalized_user_id)
platform_value = source.platform.value if source.platform is not None else None
# SimpleX: user_id is the numeric contactId (stable across renames) but
# the UI only surfaces display names, so match both. Plugin platform:
# compare by value.
if platform_value == "simplex" and source.user_name:
check_ids.add(source.user_name)
# Buzz: allowlist may hold npub or hex; inbound pubkeys are hex.
if platform_value == "buzz":
allowed_ids = _normalize_nostr_allow_entries(allowed_ids)
if user_id.startswith("npub"):
hex_user = _npub_to_hex(user_id)
if hex_user:
check_ids.add(hex_user)
return bool(check_ids & allowed_ids)
def _get_unauthorized_dm_behavior(
self,
platform: Optional[Platform],
*,
profile: Optional[str] = None,
) -> str:
"""Return how unauthorized DMs should be handled for a platform.
Resolution order:
1. Explicit per-platform ``unauthorized_dm_behavior`` in config.
2. Email -> ``"ignore"`` (inboxes hold arbitrary human mail; pairing
codes are not a safe default; matches GatewayConfig).
3. Explicit (non-default) global ``unauthorized_dm_behavior``.
4. Adapter dm_policy: ``pairing`` -> ``"pair"``; ``allowlist`` /
``disabled`` -> ``"ignore"``.
5. Any configured allowlist -> ``"ignore"`` (the operator restricted
access; spamming unknown contacts with codes is noisy and leaks).
6. Otherwise ``"pair"`` (open-gateway default).
Order: explicit per-platform config; Email → "ignore" (inboxes hold arbitrary
mail); explicit non-default global; adapter dm_policy (pairing → "pair",
allowlist/disabled → "ignore"); any configured allowlist → "ignore" (spamming
unknown contacts with codes is noisy and leaks); else "pair".
"""
config = getattr(self, "config", None)
if config and hasattr(config, "get_unauthorized_dm_behavior") and platform:
if "unauthorized_dm_behavior" in self._config_extra(platform):
return config.get_unauthorized_dm_behavior(platform)
if (
config and hasattr(config, "get_unauthorized_dm_behavior") and platform
and "unauthorized_dm_behavior" in self._config_extra(platform)
):
return config.get_unauthorized_dm_behavior(platform)
if platform == Platform.EMAIL:
return "ignore"
if config and hasattr(config, "unauthorized_dm_behavior"):
if config.unauthorized_dm_behavior != "pair":
return config.unauthorized_dm_behavior
if config and hasattr(config, "unauthorized_dm_behavior") and config.unauthorized_dm_behavior != "pair":
return config.unauthorized_dm_behavior
allowlist_keys = ["GATEWAY_ALLOWED_USERS"]
if platform:
dm_policy = self._adapter_dm_policy(platform, profile=profile)
if not dm_policy:
@@ -741,15 +606,9 @@ class GatewayAuthorizationMixin:
return "pair"
if dm_policy in {"allowlist", "disabled"}:
return "ignore"
# Historical: Yuanbao is absent from this allowlist-aware default.
env_key = "" if platform == Platform.YUANBAO else _ALLOWED_USERS_ENV.get(platform, "")
group_keys = (_GROUP_USER_ENV.get(platform), _GROUP_CHAT_ENV.get(platform))
for key in (env_key, *group_keys):
if key and _platform_gate_env(key).strip():
return "ignore"
if _platform_gate_env("GATEWAY_ALLOWED_USERS").strip():
allowlist_keys = [env_key, _GROUP_USER_ENV.get(platform), _GROUP_CHAT_ENV.get(platform), *allowlist_keys]
if any(key and _platform_gate_env(key).strip() for key in allowlist_keys):
return "ignore"
return "pair"
gateway/browser_control_artifacts.py
+58 -151
View File
@@ -1,27 +1,9 @@
"""One-shot artifact transport for browser control (Gateway side).
Transport-neutral store core for bounded browser-control artifacts
(screenshots, PDFs, uploads). It knows nothing about aiohttp: the routes in
:mod:`gateway.platforms.api_server` authenticate callers and rate-limit, then
hand bytes here. The controller WebSocket is a command channel, not a file
pipe — frames carry only ``artifact_id`` strings and the bytes live on disk
under a controlled root for a short TTL.
Contract (tests/gateway/test_browser_control_artifacts.py):
- Server-minted ``[0-9a-f]{32}`` ids resolved strictly inside the root;
client filenames are metadata only, never paths.
- Exact size and MIME caps enforced before any disk write.
- SHA-256 recorded in the receipt and re-verified by ``load``/``validate``.
- ``load`` requires the exact scope key and consumes atomically;
``validate`` checks existence/TTL/scope without consuming.
- No overwrite: an id collision is retried with a fresh id.
- ``prune_expired`` removes expired entries; the API server sweeps on demand.
Thread-safety: the in-memory index is lock-guarded; files are written to a
temp name and atomically renamed so a concurrent ``load`` never sees a
partial artifact.
"""
"""One-shot artifact transport for browser control (Gateway side); :mod:`gateway.platforms.api_server`
authenticates and rate-limits, then hands bytes here. Controller frames carry only server-minted
``[0-9a-f]{32}`` ids (client filenames are metadata, never paths); bytes live under a controlled root
for a short TTL. Size/MIME caps apply before any write; SHA-256 is re-verified on read; ``load`` needs
the exact scope key and consumes atomically. The index is lock-guarded and files are temp-written then
renamed so readers never see partials."""
from __future__ import annotations
@@ -43,10 +25,8 @@ DEFAULT_ARTIFACT_TTL_SECONDS = 300.0
DEFAULT_MAX_ARTIFACT_BYTES = 10 * 1024 * 1024
#: Exact allowlist — parameterized/unknown variants are rejected.
DEFAULT_ALLOWED_MIME_TYPES = frozenset({
"application/json", "application/pdf", "image/gif", "image/jpeg",
"image/png", "image/webp", "text/plain",
"application/json", "application/pdf", "image/gif", "image/jpeg", "image/png", "image/webp", "text/plain",
})
_ARTIFACT_ID_HEX = 32
_ARTIFACT_ID_RE = re.compile(r"^[0-9a-f]{32}$")
_TEMP_SUFFIX = ".tmp"
@@ -86,7 +66,6 @@ class ArtifactTraversal(ArtifactError):
@dataclass(frozen=True)
class ArtifactReceipt:
"""Provenance record returned to the caller of ``store``."""
artifact_id: str
sha256: str
size_bytes: int
@@ -99,35 +78,22 @@ class ArtifactReceipt:
def to_dict(self, *, download_path: str = "") -> dict[str, Any]:
"""Serialize to the wire receipt (never contains file paths)."""
receipt = {
"artifact_id": self.artifact_id,
"sha256": self.sha256,
"size_bytes": self.size_bytes,
"content_type": self.content_type,
"filename": self.filename,
"created_at": self.created_at,
"expires_at": self.expires_at,
"ttl_seconds": self.ttl_seconds,
"one_shot": True,
return {
"artifact_id": self.artifact_id, "sha256": self.sha256, "size_bytes": self.size_bytes,
"content_type": self.content_type, "filename": self.filename, "created_at": self.created_at,
"expires_at": self.expires_at, "ttl_seconds": self.ttl_seconds, "one_shot": True,
**({"download_path": download_path} if download_path else {}),
}
if download_path:
receipt["download_path"] = download_path
return receipt
def artifact_scope_key(scope: Any) -> str:
"""Derive the stable scope key an artifact is bound to.
Only principal (mandatory) + transport family participate. ``session_id``
is deliberately EXCLUDED: HTTP artifact routes authenticate by API key and
can't resolve a session, while broker dispatch always carries one — hashing
it would make upload and dispatch never compose. Ids are unguessable and
downloads one-shot, so cross-session reuse within one principal is by
design. Capabilities/optional ids are excluded so a reconnect keeps its
artifacts.
"""
principal = ""
family = ""
Only principal (mandatory) + transport family participate. ``session_id`` is deliberately EXCLUDED: HTTP
artifact routes authenticate by API key and can't resolve a session while broker dispatch always carries
one, so hashing it would make upload and dispatch never compose (ids are unguessable and downloads
one-shot). Capabilities/optional ids are excluded so a reconnect keeps its artifacts."""
principal = family = ""
try:
principal = str(getattr(scope, "principal_id", "") or "")
family = str(getattr(scope, "transport_family", "") or "")
@@ -136,12 +102,7 @@ def artifact_scope_key(scope: Any) -> str:
if not principal:
# Fail closed: only an authenticated principal may mint artifacts.
raise ArtifactError("artifact scope must carry a resolved principal")
material = f"{principal}\x00{family}".encode("utf-8")
return hashlib.sha256(material).hexdigest()
def _sha256(data: bytes) -> str:
return hashlib.sha256(data).hexdigest()
return hashlib.sha256(f"{principal}\x00{family}".encode("utf-8")).hexdigest()
@dataclass
@@ -152,14 +113,8 @@ class _ArtifactEntry:
class ArtifactStore:
"""Thread-safe, TTL-bounded, scope-bound one-shot artifact store."""
def __init__(
self, root: Path, *,
ttl_seconds: float = DEFAULT_ARTIFACT_TTL_SECONDS,
max_bytes: int = DEFAULT_MAX_ARTIFACT_BYTES,
allowed_mime_types: frozenset = DEFAULT_ALLOWED_MIME_TYPES,
clock: Optional[Callable[[], float]] = None,
) -> None:
def __init__(self, root: Path, *, ttl_seconds: float = DEFAULT_ARTIFACT_TTL_SECONDS, max_bytes: int = DEFAULT_MAX_ARTIFACT_BYTES,
allowed_mime_types: frozenset = DEFAULT_ALLOWED_MIME_TYPES, clock: Optional[Callable[[], float]] = None) -> None:
self._root = Path(root)
self._root.mkdir(parents=True, exist_ok=True)
self._ttl_seconds = max(1.0, float(ttl_seconds))
@@ -173,11 +128,7 @@ class ArtifactStore:
self._sweep_orphan_files()
def _sweep_orphan_files(self) -> None:
"""Delete on-disk files with no live index entry.
Only names matching the minted 32-hex id shape or the ``*.tmp``
staging suffix are touched; anything else in the directory is left.
"""
"""Delete on-disk files with no live index entry; only minted-id-shaped and ``*.tmp`` names are touched."""
try:
candidates = list(self._root.iterdir())
except OSError:
@@ -185,18 +136,10 @@ class ArtifactStore:
with self._lock:
live = set(self._entries)
for path in candidates:
name = path.name
is_temp = name.endswith(_TEMP_SUFFIX)
if not path.is_file() or not (is_temp or _ARTIFACT_ID_RE.fullmatch(name)):
continue
if not is_temp and name in live:
continue
with contextlib.suppress(OSError):
path.unlink(missing_ok=True)
# ------------------------------------------------------------------
# Public API
# ------------------------------------------------------------------
orphan = path.name.endswith(_TEMP_SUFFIX) or (_ARTIFACT_ID_RE.fullmatch(path.name) and path.name not in live)
if path.is_file() and orphan:
with contextlib.suppress(OSError):
path.unlink(missing_ok=True)
@property
def root(self) -> Path:
@@ -212,11 +155,7 @@ class ArtifactStore:
return self._allowed_mime_types
def store(self, data: bytes, *, filename: str, content_type: str, scope: Any) -> ArtifactReceipt:
"""Validate and store one artifact, returning its provenance receipt.
Raises :class:`ArtifactTooLarge` / :class:`ArtifactMimeRejected`
before any disk write; :class:`ArtifactError` on an unresolved scope.
"""
"""Validate and store one artifact, returning its receipt; size/MIME rejections fire before any disk write."""
size = len(data)
if size > self._max_bytes:
raise ArtifactTooLarge(f"artifact is {size} bytes; cap is {self._max_bytes}")
@@ -225,23 +164,19 @@ class ArtifactStore:
raise ArtifactMimeRejected(f"content type {content_type!r} is outside the exact allowlist")
scope_key = artifact_scope_key(scope)
now = self._clock()
# Mint a fresh id; retry on an astronomically unlikely collision.
while True:
artifact_id = secrets.token_hex(_ARTIFACT_ID_HEX // 2)
artifact_id = secrets.token_hex(16)
target = self._artifact_path(artifact_id)
with self._lock:
if artifact_id in self._entries or target.exists():
continue
receipt = ArtifactReceipt(
artifact_id=artifact_id, sha256=_sha256(data), size_bytes=size,
content_type=normalized_type, filename=_bounded_filename(filename),
created_at=now, expires_at=now + self._ttl_seconds,
ttl_seconds=self._ttl_seconds, scope_key=scope_key,
)
self._entries[artifact_id] = _ArtifactEntry(receipt=receipt, path=target)
break
if artifact_id not in self._entries and not target.exists():
receipt = ArtifactReceipt(
artifact_id=artifact_id, sha256=hashlib.sha256(data).hexdigest(), size_bytes=size,
content_type=normalized_type, filename=_bounded_filename(filename), created_at=now,
expires_at=now + self._ttl_seconds, ttl_seconds=self._ttl_seconds, scope_key=scope_key,
)
self._entries[artifact_id] = _ArtifactEntry(receipt=receipt, path=target)
break
# Temp + atomic rename so readers never observe a partial artifact.
temp = target.with_name(f"{target.name}{_TEMP_SUFFIX}")
try:
@@ -259,48 +194,39 @@ class ArtifactStore:
return receipt
def validate(self, artifact_id: str, *, scope: Any) -> ArtifactReceipt:
"""Return the receipt when the artifact is live for ``scope``
(existence, TTL, scope) without consuming it; raises otherwise."""
"""Receipt when the artifact is live for ``scope`` (existence, TTL, scope), without consuming."""
return self._entry_for(artifact_id, scope=scope).receipt
def load(self, artifact_id: str, *, scope: Any) -> tuple[bytes, ArtifactReceipt]:
"""One-shot download: verify, read, checksum, then consume.
A second ``load`` raises :class:`ArtifactNotFound`. A checksum
mismatch raises :class:`ArtifactChecksumMismatch` without consuming.
"""
"""One-shot download: verify, read, checksum, then consume (a checksum mismatch does not consume)."""
with self._lock:
entry = self._entry_for(artifact_id, scope=scope)
path = entry.path
if not path.exists():
if not entry.path.exists():
self._entries.pop(artifact_id, None)
raise ArtifactNotFound(f"artifact {artifact_id!r} is gone")
try:
data = path.read_bytes()
data = entry.path.read_bytes()
except OSError as exc:
raise ArtifactError(f"artifact read failed: {exc}") from exc
if _sha256(data) != entry.receipt.sha256:
if hashlib.sha256(data).hexdigest() != entry.receipt.sha256:
raise ArtifactChecksumMismatch(f"artifact {artifact_id!r} failed SHA-256 validation")
# Drop the index entry first so a concurrent load fails closed.
self._entries.pop(artifact_id, None)
try:
path.unlink(missing_ok=True)
entry.path.unlink(missing_ok=True)
except OSError:
logger.warning("artifact %s: file removal failed; TTL sweep will retry", artifact_id)
return data, entry.receipt
def prune_expired(self, now: Optional[float] = None) -> int:
"""Delete every artifact past its TTL (and stale temp files); return
the count removed. Idempotent."""
"""Delete every artifact past its TTL (and stale temp files); return the count removed."""
now = self._clock() if now is None else float(now)
with self._lock:
removed = self._prune_expired_locked(now)
for temp in self._root.glob(f"*{_TEMP_SUFFIX}"):
try:
with contextlib.suppress(OSError):
if temp.stat().st_mtime <= now - self._ttl_seconds:
temp.unlink(missing_ok=True)
except OSError:
continue
return removed
def count(self) -> int:
@@ -308,9 +234,11 @@ class ArtifactStore:
with self._lock:
return len(self._entries)
# ------------------------------------------------------------------
# Internals
# ------------------------------------------------------------------
def _discard_locked(self, artifact_id: str, path: Path) -> None:
"""Drop the index entry and best-effort unlink its file."""
self._entries.pop(artifact_id, None)
with contextlib.suppress(OSError):
path.unlink(missing_ok=True)
def _entry_for(self, artifact_id: str, *, scope: Any) -> _ArtifactEntry:
path = self._artifact_path(artifact_id)
@@ -326,23 +254,17 @@ class ArtifactStore:
if entry is None:
raise ArtifactNotFound(f"unknown artifact {artifact_id!r}")
if entry.receipt.expires_at <= now:
self._entries.pop(artifact_id, None)
with contextlib.suppress(OSError):
path.unlink(missing_ok=True)
self._discard_locked(artifact_id, path)
raise ArtifactExpired(f"artifact {artifact_id!r} expired")
if entry.receipt.scope_key != scope_key:
raise ArtifactScopeMismatch(f"artifact {artifact_id!r} is bound to a different scope")
return entry
def _prune_expired_locked(self, now: float) -> int:
removed = 0
for artifact_id, entry in list(self._entries.items()):
if entry.receipt.expires_at <= now:
self._entries.pop(artifact_id, None)
with contextlib.suppress(OSError):
entry.path.unlink(missing_ok=True)
removed += 1
return removed
expired = [(aid, e.path) for aid, e in self._entries.items() if e.receipt.expires_at <= now]
for artifact_id, path in expired:
self._discard_locked(artifact_id, path)
return len(expired)
def _artifact_path(self, artifact_id: str) -> Path:
"""Resolve a minted id strictly inside the controlled root."""
@@ -360,32 +282,18 @@ class ArtifactStore:
def _normalize_content_type(value: str) -> str:
"""Return the canonical MIME type, or ``""`` for malformed input."""
if not isinstance(value, str):
return ""
return value.strip().split(";", 1)[0].strip().lower()
return value.strip().split(";", 1)[0].strip().lower() if isinstance(value, str) else ""
def _bounded_filename(value: str, limit: int = 160) -> str:
"""Sanitize a display-only filename; never used as a filesystem path."""
if not isinstance(value, str):
return ""
cleaned = value.strip().replace("\\", "_").replace("/", "_")
cleaned = "".join(character for character in cleaned if ord(character) >= 32)
return cleaned[:limit]
# ----------------------------------------------------------------------
# Rate limiting (route-level, per principal)
# ----------------------------------------------------------------------
cleaned = value.strip().replace("\\", "_").replace("/", "_") if isinstance(value, str) else ""
return "".join(character for character in cleaned if ord(character) >= 32)[:limit]
class ArtifactRateLimiter:
"""Sliding-window per-key limiter; the API server keys it by principal."""
def __init__(
self, *, window_seconds: float = 60.0, max_requests: int = 30,
clock: Optional[Callable[[], float]] = None,
) -> None:
def __init__(self, *, window_seconds: float = 60.0, max_requests: int = 30, clock: Optional[Callable[[], float]] = None) -> None:
self._window_seconds = max(1.0, float(window_seconds))
self._max_requests = max(1, int(max_requests))
self._clock = clock if clock is not None else time.time
@@ -397,9 +305,8 @@ class ArtifactRateLimiter:
if not isinstance(key, str) or not key:
return False
now = self._clock()
window_start = now - self._window_seconds
with self._lock:
hits = [hit for hit in self._hits.get(key, []) if hit > window_start]
hits = [hit for hit in self._hits.get(key, []) if hit > now - self._window_seconds]
allowed = len(hits) < self._max_requests
if allowed:
hits.append(now)
gateway/browser_control_broker.py
+134 -448
View File
@@ -1,34 +1,10 @@
"""Transport-neutral browser-control broker core.
Binds an identity-scoped *controller* (the party driving a browser) to
*callers* (agents talking to it over any transport) without knowing HTTP,
WebSocket, or any wire format. The browser is a stateful single-owner
resource while the agent side is multi-tenant and multi-transport, so every
rule here makes a class of violation structurally impossible:
- Registration tickets are short-lived, single-use, identity-bound, and
``secrets``-random; ``consume_ticket`` exchanges one exactly once. The
ticket is the only cross-transport credential minted here; transports
decide how to carry it.
- ``select`` matches on *every* stable identity field (principal, profile,
session, controller id, browser profile id, transport family) plus the
controller's currently negotiated capability set; partial matches yield
``None``. A same-identity reconnect renegotiates capabilities in place.
- One pending command per command id; ``complete`` is single-shot (late
completion after cancel/detach returns ``False``).
- ``cancel`` aborts only the command matching scope + tool_call_id.
- ``detach`` fails closed: pending work of that scope raises
:class:`ControllerCancelled` instead of racing a late ``complete``.
- ``disconnect`` marks a transport offline without cancelling running work;
re-attaching the same identity refreshes the callback and may still
deliver the original result. Detach, cancel, identity replacement, and
timeout remain terminal.
Thread-safety: all state transitions happen under one reentrant lock; the
send callback is invoked *outside* it so a controller may synchronously
``complete`` from inside its own send, and waiters park on per-command
events, never on the broker lock.
"""
"""Transport-neutral browser-control broker core: binds an identity-scoped *controller*
(the party driving a browser) to *callers* on any transport. Tickets are short-lived,
single-use, identity-bound and consumed exactly once; ``select`` matches every stable
identity field plus the current capability set; ``complete`` is single-shot; ``detach``
fails pending work closed while ``disconnect`` only marks the transport offline. State
changes happen under one RLock; the send callback runs *outside* it so a controller may
``complete`` from inside its own send."""
from __future__ import annotations
@@ -48,39 +24,18 @@ DEFAULT_TICKET_TTL = 30.0
DEFAULT_COMMAND_TIMEOUT = 30.0
#: Maximum cancel frames retained while a same-identity controller is offline.
MAX_DEFERRED_CANCELS = 512
#: Current wire protocol version. Registration requires this exact integer;
#: booleans are rejected even though ``bool`` subclasses ``int``.
#: Current wire protocol version; registration requires this exact int (bools rejected).
BROWSER_CONTROL_PROTOCOL_VERSION = 1
#: Exact controller capability allowlist shared by every transport. Raw CDP,
#: script evaluation, console access, and other privileged surfaces stay out.
BROWSER_CONTROL_CAPABILITIES = frozenset(
{
"controller.noop",
"browser_back",
"browser_click",
"browser_navigate",
"browser_press",
"browser_screenshot",
"browser_scroll",
"browser_snapshot",
"browser_tab_activate",
"browser_tabs",
"browser_type",
}
)
#: Privileged capabilities (JS eval, raw CDP): fail-closed unless Developer
#: Mode (``browser.extension_control.developer_mode``) is on AND the
#: controller explicitly negotiated them.
#: Exact controller capability allowlist shared by every transport (raw CDP/eval/console stay out).
BROWSER_CONTROL_CAPABILITIES = frozenset({
"controller.noop", "browser_back", "browser_click", "browser_navigate", "browser_press", "browser_screenshot",
"browser_scroll", "browser_snapshot", "browser_tab_activate", "browser_tabs", "browser_type",
})
#: Privileged capabilities: fail-closed unless Developer Mode is on AND explicitly negotiated.
BROWSER_CONTROL_DEVELOPER_CAPABILITIES = frozenset({"browser_cdp", "browser_evaluate"})
#: Artifact transport capabilities. Non-developer because payloads never ride
#: in controller frames: only a store-validated ``artifact_id`` is dispatched.
BROWSER_CONTROL_ARTIFACT_CAPABILITIES = frozenset(
{"browser_artifact_download", "browser_artifact_upload"}
)
#: Artifact transport capabilities; non-developer because only a store-validated ``artifact_id`` travels.
BROWSER_CONTROL_ARTIFACT_CAPABILITIES = frozenset({"browser_artifact_download", "browser_artifact_upload"})
#: Wire method names for controller frames; transports carry them verbatim.
FRAME_COMMAND = "browser.controller.command"
@@ -96,22 +51,14 @@ def _extension_control_flag(config: Optional[dict], key: str) -> bool:
"""Read ``browser.extension_control.<key>`` as a literal ``True`` (default off)."""
if config is None:
try:
# Hot path (every browser tool call / check_fn): the read-only
# loader skips load_config()'s deepcopy; we never mutate.
# Hot path (every browser tool call): the read-only loader skips load_config()'s deepcopy.
from hermes_cli.config import load_config_readonly
config = load_config_readonly()
except Exception:
return False
if not isinstance(config, dict):
return False
browser = config.get("browser")
if not isinstance(browser, dict):
return False
extension_control = browser.get("extension_control")
if not isinstance(extension_control, dict):
return False
return extension_control.get(key, False) is True
browser = config.get("browser") if isinstance(config, dict) else None
extension_control = browser.get("extension_control") if isinstance(browser, dict) else None
return isinstance(extension_control, dict) and extension_control.get(key, False) is True
def browser_control_developer_mode(config: Optional[dict] = None) -> bool:
@@ -124,25 +71,13 @@ def browser_control_enabled(config: Optional[dict] = None) -> bool:
return _extension_control_flag(config, "enabled")
def filter_browser_control_capabilities(
value: Any,
*,
developer_mode: Optional[bool] = None,
) -> frozenset:
"""Return the permitted subset of a JSON/RPC capability list.
Non-list input has no capabilities; unknown/non-string entries are
dropped (registration rejects an empty returned set). Developer
capabilities pass only when Developer Mode is on (passed in, else read
from live config).
"""
def filter_browser_control_capabilities(value: Any, *, developer_mode: Optional[bool] = None) -> frozenset:
"""Permitted subset of a capability list (non-list -> empty); developer caps only in Developer Mode."""
if not isinstance(value, list):
return frozenset()
allowed = BROWSER_CONTROL_CAPABILITIES | BROWSER_CONTROL_ARTIFACT_CAPABILITIES
if developer_mode is None:
developer_mode = browser_control_developer_mode()
if developer_mode is True:
allowed = allowed | BROWSER_CONTROL_DEVELOPER_CAPABILITIES
if (browser_control_developer_mode() if developer_mode is None else developer_mode) is True:
allowed |= BROWSER_CONTROL_DEVELOPER_CAPABILITIES
return frozenset(c for c in value if isinstance(c, str) and c in allowed)
@@ -173,7 +108,6 @@ class ControllerRejected(BrowserControlError):
@dataclass(frozen=True)
class ControllerScope:
"""Exact controller identity plus capability set; equality is over all fields."""
principal_id: Optional[str] = None
profile_id: Optional[str] = None
session_id: Optional[str] = None
@@ -183,26 +117,17 @@ class ControllerScope:
capabilities: frozenset = frozenset()
def _scope_identity(scope: ControllerScope) -> tuple:
"""Stable identity fields, excluding negotiated capabilities."""
return (
scope.principal_id,
scope.profile_id,
scope.session_id,
scope.controller_id,
scope.browser_profile_id,
scope.transport_family,
)
#: Stable identity fields; negotiated ``capabilities`` are deliberately excluded.
_IDENTITY_FIELDS = ("principal_id", "profile_id", "session_id", "controller_id", "browser_profile_id", "transport_family")
def _same_scope_identity(first: ControllerScope, second: ControllerScope) -> bool:
return _scope_identity(first) == _scope_identity(second)
return all(getattr(first, name) == getattr(second, name) for name in _IDENTITY_FIELDS)
@dataclass(frozen=True)
class Ticket:
"""Opaque, single-use registration credential."""
value: str
expires_at: float
@@ -221,9 +146,7 @@ class _Controller:
owner: Any = None
connected: bool = True
deferred_cancels: list[dict] = field(default_factory=list)
# Serializes command/cancel writes with detach or replacement. Broker
# state is never held while waiting on it, so a transport callback may
# synchronously complete() without deadlocking.
# Serializes command/cancel writes with detach or replacement; never held with broker state.
send_lock: threading.Lock = field(default_factory=threading.Lock)
@@ -239,20 +162,14 @@ class _PendingCommand:
result: Any = None
def _cancel_frame(pending: _PendingCommand) -> dict:
return {"method": FRAME_CANCEL, "params": {"command_id": pending.command_id, "tool_call_id": pending.tool_call_id}}
class BrowserControlBroker:
"""Thread-safe broker core binding controllers to callers.
``clock`` is injectable (default ``time.monotonic``) so tests pin expiry.
"""
def __init__(
self,
*,
ticket_ttl: float = DEFAULT_TICKET_TTL,
command_timeout: float = DEFAULT_COMMAND_TIMEOUT,
clock: Optional[Callable[[], float]] = None,
developer_mode: Optional[bool] = None,
) -> None:
"""Thread-safe broker core; ``clock`` is injectable (default ``time.monotonic``)."""
def __init__(self, *, ticket_ttl: float = DEFAULT_TICKET_TTL, command_timeout: float = DEFAULT_COMMAND_TIMEOUT,
clock: Optional[Callable[[], float]] = None, developer_mode: Optional[bool] = None) -> None:
self._ticket_ttl = ticket_ttl
self._command_timeout = command_timeout
self._clock = clock if clock is not None else time.monotonic
@@ -260,19 +177,15 @@ class BrowserControlBroker:
self._tickets: Dict[str, _TicketRecord] = {}
self._controllers: Dict[ControllerScope, _Controller] = {}
self._pending: Dict[str, _PendingCommand] = {}
# None defers to live config on every selection so flipping
# developer_mode off REVOKES raw CDP/eval from attached controllers
# without restart; an explicit bool pins the gate (tests, multi-tenant).
self._developer_mode_pinned: Optional[bool] = (
None if developer_mode is None else developer_mode is True
)
# None defers to live config on every selection (so flipping developer_mode off REVOKES
# raw CDP/eval from attached controllers without restart); a bool pins the gate.
self._developer_mode_pinned: Optional[bool] = None if developer_mode is None else developer_mode is True
# Artifact stores keyed by profile id; ``None`` is the default slot.
# Per-profile stores keep multiplex profile A from pinning profile B
# to A's physical root.
self._artifact_stores: Dict[Optional[str], Any] = {}
def _developer_mode_now(self) -> bool:
"""Current Developer Mode authority (live config unless pinned)."""
@property
def developer_mode(self) -> bool:
"""Whether privileged capabilities may be selected/dispatched (live config unless pinned)."""
if self._developer_mode_pinned is not None:
return self._developer_mode_pinned
try:
@@ -280,55 +193,29 @@ class BrowserControlBroker:
except Exception:
return False
@property
def developer_mode(self) -> bool:
"""Whether privileged capabilities may be selected/dispatched."""
return self._developer_mode_now()
def attach_artifact_store(
self, store: Any, *, profile_id: Optional[str] = None
) -> None:
"""Attach a store exposing ``validate(artifact_id, *, scope) -> receipt``
(raising the artifacts module's :class:`ArtifactError` subclasses).
``profile_id`` scopes the store to one profile on multiplex hosts;
``None`` registers the default store. ``store=None`` clears the slot.
Artifact actions without a resolvable store fail closed.
"""
def attach_artifact_store(self, store: Any, *, profile_id: Optional[str] = None) -> None:
"""Attach a store exposing ``validate(artifact_id, *, scope) -> receipt`` for one profile
(``None`` = default slot); ``store=None`` clears the slot. Artifact actions fail closed without one."""
if store is None:
self._artifact_stores.pop(profile_id, None)
return
self._artifact_stores[profile_id] = store
else:
self._artifact_stores[profile_id] = store
def _artifact_store_for_scope(self, scope: "ControllerScope") -> Any:
"""Exact profile-scoped store, else the default (``None``) slot so
single-profile hosts keep the historical one-store behaviour."""
profile = getattr(scope, "profile_id", None) or None
store = self._artifact_stores.get(profile)
store = self._artifact_stores.get(getattr(scope, "profile_id", None) or None)
return store if store is not None else self._artifact_stores.get(None)
# ------------------------------------------------------------------
# Registration tickets
# ------------------------------------------------------------------
def mint_ticket(self, scope: ControllerScope) -> Ticket:
"""Mint a short-lived, single-use ticket bound to ``scope``."""
now = self._clock()
with self._lock:
for stale in [v for v, rec in self._tickets.items() if rec.expires_at <= now]:
del self._tickets[stale]
self._tickets = {v: rec for v, rec in self._tickets.items() if rec.expires_at > now}
value = secrets.token_urlsafe(32)
record = _TicketRecord(scope=scope, expires_at=now + self._ticket_ttl)
self._tickets[value] = record
self._tickets[value] = record = _TicketRecord(scope=scope, expires_at=now + self._ticket_ttl)
return Ticket(value=value, expires_at=record.expires_at)
def consume_ticket(self, value: str) -> ControllerScope:
"""Exchange a ticket for its scope exactly once.
Raises :class:`ControllerTicketInvalid` for unknown, already-consumed,
or expired tickets; expiry is checked against the live clock at
consume time.
"""
"""Exchange a ticket for its scope exactly once; unknown/consumed/expired -> ControllerTicketInvalid."""
now = self._clock()
with self._lock:
record = self._tickets.get(value)
@@ -341,52 +228,26 @@ class BrowserControlBroker:
record.consumed = True
return record.scope
# ------------------------------------------------------------------
# Controller registration / selection
# ------------------------------------------------------------------
def _controller_for_identity_locked(self, scope: ControllerScope) -> Optional[_Controller]:
"""Attached controller sharing ``scope``'s stable identity (any capabilities)."""
return next(
(c for c in self._controllers.values() if _same_scope_identity(c.scope, scope)),
None,
)
return next((c for c in self._controllers.values() if _same_scope_identity(c.scope, scope)), None)
def _lane_scopes_locked(self, session_id: str, principal_id: str, family: str) -> list[ControllerScope]:
"""Attached scopes bound to one session lane (session + principal + transport)."""
return [
scope
for scope in self._controllers
if scope.session_id == session_id
and scope.principal_id == principal_id
and scope.transport_family == family
]
def _live_controller(self, scope: ControllerScope) -> Optional[_Controller]:
with self._lock:
controller = self._controller_for_identity_locked(scope)
return controller if controller is not None and controller.connected else None
def attach(
self,
scope: ControllerScope,
send: Callable[[dict], None],
*,
owner: Any = None,
) -> None:
"""Attach or refresh the controller for one stable identity.
A same-identity reconnect refreshes the send callback and negotiated
capabilities without cancelling pending work. A different controller
or browser profile in the same authenticated session lane
hard-replaces the previous identity.
"""
def attach(self, scope: ControllerScope, send: Callable[[dict], None], *, owner: Any = None) -> None:
"""Attach or refresh the controller for one stable identity: a same-identity reconnect refreshes send
and capabilities without cancelling pending work; a different controller/browser profile in the same
authenticated session lane hard-replaces it."""
while True:
with self._lock:
existing = self._controller_for_identity_locked(scope)
lane_scopes = [
candidate
for candidate in self._controllers
if candidate.principal_id == scope.principal_id
and candidate.profile_id == scope.profile_id
and candidate.session_id == scope.session_id
and candidate.transport_family == scope.transport_family
and not _same_scope_identity(candidate, scope)
c for c in self._controllers if not _same_scope_identity(c, scope)
and (c.principal_id, c.profile_id, c.session_id, c.transport_family)
== (scope.principal_id, scope.profile_id, scope.session_id, scope.transport_family)
]
if existing is None and not lane_scopes:
self._controllers[scope] = _Controller(scope=scope, send=send, owner=owner)
@@ -404,69 +265,41 @@ class BrowserControlBroker:
if self._controllers.get(existing.scope) is not existing:
continue
self._controllers.pop(existing.scope, None)
existing.scope = scope
existing.send = send
existing.owner = owner
existing.connected = False
for pending in self._pending.values():
if _same_scope_identity(pending.scope, scope):
pending.scope = scope
deferred = list(existing.deferred_cancels)
existing.deferred_cancels.clear()
existing.scope, existing.send, existing.owner, existing.connected = scope, send, owner, False
for pending in self._pending_for_scope_locked(scope):
pending.scope = scope
deferred, existing.deferred_cancels = existing.deferred_cancels, []
self._controllers[scope] = existing
unsent: list[dict] = []
for index, frame in enumerate(deferred):
try:
send(frame)
continue
except Exception:
logger.exception("failed to flush deferred browser-controller cancel")
unsent = deferred[index:]
break
if unsent:
with self._lock:
if self._controllers.get(scope) is existing:
existing.deferred_cancels = unsent[-MAX_DEFERRED_CANCELS:]
raise ConnectionError(
"browser controller reconnect could not flush deferred cancels"
)
existing.deferred_cancels = deferred[index:][-MAX_DEFERRED_CANCELS:]
raise ConnectionError("browser controller reconnect could not flush deferred cancels")
with self._lock:
if self._controllers.get(scope) is existing:
existing.connected = True
return
def select(self, scope: ControllerScope, capability: str) -> Optional[_Controller]:
"""Return the connected controller matching identity and capability.
The caller's capabilities are not authoritative: identity is matched,
then the controller's *current* negotiated set is checked. Offline
controllers never accept new dispatches. Developer capabilities are
additionally gated on the LIVE Developer Mode flag (unless pinned).
"""
if capability in BROWSER_CONTROL_DEVELOPER_CAPABILITIES and not self._developer_mode_now():
"""Connected controller matching identity whose *current* negotiated set holds ``capability`` (the
caller's set is not authoritative); developer capabilities are also gated on LIVE Developer Mode."""
if capability in BROWSER_CONTROL_DEVELOPER_CAPABILITIES and not self.developer_mode:
return None
with self._lock:
controller = self._controller_for_identity_locked(scope)
if controller is None or not controller.connected:
return None
return controller if capability in controller.scope.capabilities else None
controller = self._live_controller(scope)
return controller if controller is not None and capability in controller.scope.capabilities else None
def is_owner(self, scope: ControllerScope, owner: Any) -> bool:
"""Whether ``owner`` is the exact live transport for ``scope``.
"""Whether ``owner`` is the exact live transport for ``scope`` (capability-independent)."""
controller = self._live_controller(scope)
return controller is not None and controller.owner is owner
Independent of capabilities, so a least-privilege controller need not
request ``controller.noop`` merely to heartbeat or complete an action.
"""
with self._lock:
controller = self._controller_for_identity_locked(scope)
return controller is not None and controller.connected and controller.owner is owner
def disconnect(
self,
scope: ControllerScope,
*,
owner: Any = _OWNER_UNSET,
) -> bool:
def disconnect(self, scope: ControllerScope, *, owner: Any = _OWNER_UNSET) -> bool:
"""Mark one exact controller transport offline without cancelling work."""
with self._lock:
controller = self._controller_for_identity_locked(scope)
@@ -474,37 +307,21 @@ class BrowserControlBroker:
return False
with controller.send_lock:
with self._lock:
if self._controllers.get(controller.scope) is not controller:
owned = owner is _OWNER_UNSET or controller.owner is owner
if self._controllers.get(controller.scope) is not controller or not owned:
return False
if owner is not _OWNER_UNSET and controller.owner is not owner:
return False
controller.connected = False
controller.owner = None
controller.connected, controller.owner = False, None
return True
def detach(
self,
scope: ControllerScope,
*,
owner: Any = _OWNER_UNSET,
notify_controller: bool = True,
) -> None:
"""Remove the controller for ``scope`` and fail its pending work closed.
Pending commands resolve cancelled (dispatchers raise
:class:`ControllerCancelled`); a late ``complete`` returns ``False``.
"""
def detach(self, scope: ControllerScope, *, owner: Any = _OWNER_UNSET, notify_controller: bool = True) -> None:
"""Remove the controller for ``scope`` and fail its pending work closed (ControllerCancelled)."""
with self._lock:
controller = self._controllers.get(scope)
if controller is None:
return
if owner is not _OWNER_UNSET and controller.owner != owner:
if controller is None or (owner is not _OWNER_UNSET and controller.owner != owner):
return
with controller.send_lock:
with self._lock:
if self._controllers.get(scope) is not controller:
return
if owner is not _OWNER_UNSET and controller.owner != owner:
if self._controllers.get(scope) is not controller or (owner is not _OWNER_UNSET and controller.owner != owner):
return
self._controllers.pop(scope, None)
pendings = self._pending_for_scope_locked(scope)
@@ -515,65 +332,31 @@ class BrowserControlBroker:
if notify_controller:
self._emit_cancel_frames(controller, pendings)
# ------------------------------------------------------------------
# Command lifecycle
# ------------------------------------------------------------------
def dispatch(
self,
scope: ControllerScope,
*,
action: str,
arguments: Optional[dict] = None,
tool_call_id: Optional[str] = None,
self, scope: ControllerScope, *, action: str, arguments: Optional[dict] = None, tool_call_id: Optional[str] = None,
) -> Any:
"""Send one controller command and block for its completion.
Raises :class:`ControllerUnavailable` (no exact match),
:class:`ControllerCancelled`, :class:`ControllerTimeout`, or
:class:`ControllerRejected` (``ok=False``). Artifact actions also
require an attached store and an approved ``artifact_id`` argument;
only the id travels in the frame, never the payload.
"""
"""Send one controller command and block for completion; raises ControllerUnavailable/Cancelled/Timeout/
Rejected. Artifact actions also need an attached store and an approved ``artifact_id`` (only the id travels)."""
controller = self.select(scope, action)
if controller is None:
raise ControllerUnavailable(
f"no controller for scope {scope!r} with capability {action!r}"
)
raise ControllerUnavailable(f"no controller for scope {scope!r} with capability {action!r}")
arguments = dict(arguments or {})
if action in BROWSER_CONTROL_ARTIFACT_CAPABILITIES:
self._validate_artifact_reference(scope, action, arguments)
command_id = secrets.token_hex(16)
frame = {
"method": FRAME_COMMAND,
"params": {
"command_id": command_id,
"action": action,
"arguments": arguments,
"controller_id": scope.controller_id,
"browser_profile_id": scope.browser_profile_id,
"tool_call_id": tool_call_id,
},
}
pending = _PendingCommand(
scope=controller.scope,
command_id=command_id,
tool_call_id=tool_call_id,
)
frame = {"method": FRAME_COMMAND, "params": {
"command_id": command_id, "action": action, "arguments": arguments, "controller_id": scope.controller_id,
"browser_profile_id": scope.browser_profile_id, "tool_call_id": tool_call_id,
}}
pending = _PendingCommand(scope=controller.scope, command_id=command_id, tool_call_id=tool_call_id)
with controller.send_lock:
with self._lock:
# select() ran outside the send lock; revalidate the live
# controller so disconnect/replacement can't strand a command.
attached = self._controller_for_identity_locked(scope)
if attached is not controller or not controller.connected:
raise ControllerUnavailable(
f"controller for scope {scope!r} detached before dispatch"
)
if self._controller_for_identity_locked(scope) is not controller or not controller.connected:
raise ControllerUnavailable(f"controller for scope {scope!r} detached before dispatch")
pending.scope = controller.scope
self._pending[command_id] = pending
try:
controller.send(frame)
except Exception:
@@ -583,14 +366,13 @@ class BrowserControlBroker:
raise
if not pending.event.wait(timeout=self._command_timeout):
timed_out = False
with self._lock:
# Event.wait() may return False at the exact boundary where a
# completion already won and removed the pending command.
if not pending.done and self._pending.get(command_id) is pending:
timed_out = not pending.done and self._pending.get(command_id) is pending
if timed_out:
pending.done = True
del self._pending[command_id]
timed_out = True
if timed_out:
with controller.send_lock:
with self._lock:
@@ -600,97 +382,48 @@ class BrowserControlBroker:
active = None
if active is not None:
self._emit_cancel_frames(active, [pending])
raise ControllerTimeout(
f"controller did not complete command {command_id!r} "
f"within {self._command_timeout}s"
)
raise ControllerTimeout(f"controller did not complete command {command_id!r} within {self._command_timeout}s")
if pending.cancelled:
raise ControllerCancelled(f"command {command_id!r} was cancelled")
if not pending.ok:
raise ControllerRejected(
f"controller rejected command {command_id!r}: {pending.result!r}"
)
raise ControllerRejected(f"controller rejected command {command_id!r}: {pending.result!r}")
return pending.result
def complete(
self,
command_id: str,
*,
scope: Optional[ControllerScope] = None,
ok: bool,
result: Any = None,
) -> bool:
"""Resolve a pending command by id; ``False`` when none is pending.
Safe from inside the controller's own ``send`` callback. Late
completions after ``cancel``/``detach`` are ignored (``False``).
"""
def complete(self, command_id: str, *, scope: Optional[ControllerScope] = None, ok: bool, result: Any = None) -> bool:
"""Resolve a pending command by id; ``False`` when none is pending. Safe from inside the send callback."""
with self._lock:
pending = self._pending.get(command_id)
if pending is None or pending.done:
if pending is None or pending.done or (scope is not None and pending.scope != scope):
return False
if scope is not None and pending.scope != scope:
return False
pending.done = True
pending.ok = ok is True
pending.result = result
pending.done, pending.ok, pending.result = True, ok is True, result
del self._pending[command_id]
pending.event.set()
return True
def cancel(self, scope: ControllerScope, *, tool_call_id: Optional[str]) -> bool:
"""Cancel exactly the pending command matching ``scope`` + tool_call_id.
Emits one cancel frame; returns ``False`` when nothing matched so
transports can answer idempotently.
"""
with self._lock:
controller = self._controller_for_identity_locked(scope)
if controller is None or not controller.connected:
"""Cancel the pending command matching ``scope`` + tool_call_id (one cancel frame); ``False`` if none."""
controller = self._live_controller(scope)
if controller is None:
return False
with controller.send_lock:
with self._lock:
attached = self._controller_for_identity_locked(scope)
if attached is not controller or not controller.connected:
if self._controller_for_identity_locked(scope) is not controller or not controller.connected:
return False
target = next(
(
p
for p in self._pending.values()
if _same_scope_identity(p.scope, scope)
and p.tool_call_id == tool_call_id
and not p.done
),
None,
)
target = next((p for p in self._pending_for_scope_locked(scope)
if p.tool_call_id == tool_call_id and not p.done), None)
if target is None:
return False
self._resolve_pending(target, cancelled=True)
self._emit_cancel_frames(controller, [target])
return True
# ------------------------------------------------------------------
# Internals (callers hold the lock unless noted)
# ------------------------------------------------------------------
def _resolve_pending(self, pending: _PendingCommand, *, cancelled: bool) -> None:
pending.cancelled = cancelled
pending.done = True
pending.cancelled, pending.done = cancelled, True
del self._pending[pending.command_id]
pending.event.set()
def _validate_artifact_reference(
self,
scope: ControllerScope,
action: str,
arguments: dict,
) -> None:
"""Fail closed unless ``arguments`` carries a store-approved artifact id.
Missing store/id, traversal, expiry, checksum, or scope mismatch all
surface as :class:`ControllerRejected` before any frame is emitted.
"""
def _validate_artifact_reference(self, scope: ControllerScope, action: str, arguments: dict) -> None:
"""Fail closed unless ``arguments`` carries a store-approved artifact id (failures -> ControllerRejected)."""
store = self._artifact_store_for_scope(scope)
if store is None:
raise ControllerRejected(f"{action} requires an attached artifact store")
@@ -702,22 +435,10 @@ class BrowserControlBroker:
except ControllerRejected:
raise
except Exception as exc:
raise ControllerRejected(
f"{action} rejected artifact reference {artifact_id!r}: {exc}"
) from exc
@staticmethod
def _cancel_frame(pending: _PendingCommand) -> dict:
return {
"method": FRAME_CANCEL,
"params": {
"command_id": pending.command_id,
"tool_call_id": pending.tool_call_id,
},
}
raise ControllerRejected(f"{action} rejected artifact reference {artifact_id!r}: {exc}") from exc
def _defer_cancel_locked(self, controller: _Controller, pending: _PendingCommand) -> None:
controller.deferred_cancels.append(self._cancel_frame(pending))
controller.deferred_cancels.append(_cancel_frame(pending))
if len(controller.deferred_cancels) > MAX_DEFERRED_CANCELS:
del controller.deferred_cancels[:-MAX_DEFERRED_CANCELS]
@@ -728,65 +449,30 @@ class BrowserControlBroker:
"""Send cancel frames (caller holds ``send_lock``, never the broker lock)."""
for pending in pendings:
try:
controller.send(self._cancel_frame(pending))
controller.send(_cancel_frame(pending))
except Exception:
logger.exception(
"failed to emit cancel frame for command %r", pending.command_id
)
logger.exception("failed to emit cancel frame for command %r", pending.command_id)
# ------------------------------------------------------------------
# Session-lane lookups / bulk teardown
# ------------------------------------------------------------------
@staticmethod
def _lane_key(session_id, task_id, principal_id, transport_family) -> Optional[tuple[str, str, str]]:
target = str(session_id or task_id or "").strip()
principal = str(principal_id or "").strip()
family = str(transport_family or "").strip()
if not target or not principal or not family:
return None
return target, principal, family
def scope_for_session(
self,
*,
session_id: Optional[str] = None,
task_id: Optional[str] = None,
principal_id: Optional[str] = None,
transport_family: Optional[str] = None,
) -> Optional[ControllerScope]:
"""Return one unambiguous attached scope for a server-owned session.
The public session id is only a hint; the caller must also supply its
server-derived principal and transport family. Missing identity, no
match, or multiple matches fail closed rather than picking by order.
"""
key = self._lane_key(session_id, task_id, principal_id, transport_family)
if key is None:
return None
def _lane_scopes(self, session_id, task_id, principal_id, transport_family) -> list[ControllerScope]:
"""Attached scopes bound to one session lane (session + principal + transport)."""
key = tuple(str(v or "").strip() for v in (session_id or task_id, principal_id, transport_family))
if not all(key):
return []
with self._lock:
matches = self._lane_scopes_locked(*key)
return [s for s in self._controllers if (s.session_id, s.principal_id, s.transport_family) == key]
def scope_for_session(self, *, session_id: Optional[str] = None, task_id: Optional[str] = None,
principal_id: Optional[str] = None, transport_family: Optional[str] = None) -> Optional[ControllerScope]:
"""One unambiguous attached scope for a server-owned session (session id is only a hint; the caller
supplies its server-derived principal + transport family). Missing/ambiguous identity fails closed."""
matches = self._lane_scopes(session_id, task_id, principal_id, transport_family)
return matches[0] if len(matches) == 1 else None
def lane_registered(
self,
*,
session_id: Optional[str] = None,
task_id: Optional[str] = None,
principal_id: Optional[str] = None,
transport_family: Optional[str] = None,
) -> bool:
"""Whether ANY controller (even offline) registered for this lane.
Distinguishes "lane bound but currently unavailable" (fail closed;
the extension lane stays authoritative) from "never registered"
(caller keeps the legacy backend). Ambiguous lanes report True.
"""
key = self._lane_key(session_id, task_id, principal_id, transport_family)
if key is None:
return False
with self._lock:
return bool(self._lane_scopes_locked(*key))
def lane_registered(self, *, session_id: Optional[str] = None, task_id: Optional[str] = None,
principal_id: Optional[str] = None, transport_family: Optional[str] = None) -> bool:
"""Whether ANY controller (even offline) registered for this lane: "bound but unavailable" fails closed
vs "never registered" (caller keeps the legacy backend). Ambiguous lanes report True."""
return bool(self._lane_scopes(session_id, task_id, principal_id, transport_family))
def disconnect_owner(self, owner: Any) -> int:
"""Mark every controller owned by one lost transport offline."""
gateway/cgroup_cleanup.py
+7 -12
View File
@@ -3,15 +3,14 @@
Runs as ``ExecStopPost=`` after the gateway's main process has exited: the
safety net for long-lived helpers the gateway doesn't track (``adb``, platform
bridges) that would otherwise be orphaned in the cgroup and block
``Restart=always``.
Per-PID SIGKILLs over ``cgroup.procs`` are used deliberately instead of writing
``1`` to ``cgroup.kill``: the kernel has returned ``EINVAL`` on the cgroup-wide
kill while per-PID signal delivery still works.
``Restart=always``. Per-PID SIGKILLs over ``cgroup.procs`` are used instead of
writing ``1`` to ``cgroup.kill``: the kernel has returned ``EINVAL`` on the
cgroup-wide kill while per-PID signal delivery still works.
"""
from __future__ import annotations
import contextlib
import os
import re
import signal
@@ -36,23 +35,19 @@ def _read_cgroup_pids(cgroup_path: str) -> list[int]:
return []
pids: list[int] = []
for line in raw.splitlines():
try:
with contextlib.suppress(ValueError):
pids.append(int(line.strip()))
except ValueError:
continue
return pids
def reap_cgroup(cgroup_path: str | None = None) -> int:
"""SIGKILL every PID in the cgroup other than the caller. Returns the count killed."""
if cgroup_path is None:
cgroup_path = _own_cgroup_path()
cgroup_path = _own_cgroup_path() if cgroup_path is None else cgroup_path
if not cgroup_path:
return 0
own = os.getpid()
killed = 0
for pid in _read_cgroup_pids(cgroup_path):
if pid == own:
if pid == os.getpid():
continue
try:
os.kill(pid, signal.SIGKILL) # windows-footgun: ok — Linux-only (reads /proc, /sys/fs/cgroup; runs from a systemd unit)
gateway/channel_directory.py
+140 -170
View File
@@ -5,6 +5,7 @@ send_message reads it for action="list" and to resolve friendly channel names to
"""
import asyncio
import contextlib
import json
import logging
import time
@@ -75,12 +76,10 @@ def _apply_channel_aliases(platforms: Dict[str, Any]) -> None:
continue
chat_id = str(chat_id)
friendly = friendly.strip()
matched = False
for e in entries:
if isinstance(e, dict) and e.get("id") == chat_id:
e["name"] = friendly
matched = True
if not matched:
matches = [e for e in entries if isinstance(e, dict) and e.get("id") == chat_id]
for e in matches:
e["name"] = friendly
if not matches:
entries.append({
"id": chat_id, "name": friendly,
"type": "group" if chat_id.endswith("@g.us") else "dm", "thread_id": None,
@@ -92,7 +91,7 @@ def _normalize_channel_query(value: str) -> str:
def _channel_target_name(platform_name: str, channel: Dict[str, Any]) -> str:
"""Return the human-facing target label shown to users for a channel entry."""
"""Human-facing target label for a channel entry."""
name = channel["name"]
if platform_name == "discord" and channel.get("guild"):
return f"#{name}"
@@ -118,8 +117,11 @@ def _session_entry_name(origin: Dict[str, Any]) -> str:
return f"{base_name} / {topic_label}"
def _warn_slack_directory(team_id: str, detail: str) -> None:
"""Warn once per team/error per interval for recurring Slack refresh failures."""
def _report_slack_failure(team_id: str, error_code: Optional[str], detail: str) -> None:
"""missing_scope is expected (session-history fallback); anything else warns once per interval."""
if error_code == "missing_scope":
logger.debug("Channel directory: Slack team %s lacks channels:read; using session history only", team_id)
return
key = (str(team_id), str(detail))
now = time.monotonic()
last = _slack_directory_warning_last.get(key)
@@ -130,14 +132,6 @@ def _warn_slack_directory(team_id: str, detail: str) -> None:
logger.debug("Channel directory: suppressed repeated Slack channel list failure for team %s: %s", team_id, detail)
def _report_slack_failure(team_id: str, error_code: Optional[str], detail: str) -> None:
"""missing_scope is expected (session-history fallback); anything else warns."""
if error_code == "missing_scope":
logger.debug("Channel directory: Slack team %s lacks channels:read; using session history only", team_id)
else:
_warn_slack_directory(team_id, detail)
# --- Build / refresh -------------------------------------------------------
async def build_channel_directory(adapters: Dict[Any, Any]) -> Dict[str, Any]:
@@ -178,12 +172,10 @@ async def build_channel_directory(adapters: Dict[Any, Any]) -> Dict[str, Any]:
for plat in Platform:
await _discover(plat.value)
# Plugin platforms are dynamic enum members missing from Platform.__members__.
try:
with contextlib.suppress(Exception):
from gateway.platform_registry import platform_registry
for entry in platform_registry.plugin_entries():
await _discover(entry.name)
except Exception:
pass
_apply_channel_aliases(platforms)
@@ -222,24 +214,18 @@ def _build_discord(adapter) -> List[Dict[str, str]]:
def _slack_api_error_code(error: Exception) -> Optional[str]:
"""Return Slack Web API error code from SlackApiError-like exceptions."""
response = getattr(error, "response", None)
if response is None:
return None
try:
value = response.get("error")
except Exception:
return None
return str(value) if value else None
"""Slack Web API error code from SlackApiError-like exceptions."""
with contextlib.suppress(Exception):
value = error.response.get("error")
return str(value) if value else None
return None
def _normalize_adapter_channels(raw_channels: Any) -> List[Dict[str, Any]]:
"""Validate and dedupe entries returned by an adapter's ``list_channels()`` hook."""
channels: List[Dict[str, Any]] = []
seen_ids = set()
if not isinstance(raw_channels, list):
return channels
for raw in raw_channels:
for raw in raw_channels if isinstance(raw_channels, list) else ():
if not isinstance(raw, dict):
continue
channel_id = str(raw.get("id") or "").strip()
@@ -260,47 +246,86 @@ def _slack_base_id(entry_id: str) -> str:
return entry_id.split(":", 1)[0]
async def _build_slack(adapter) -> List[Dict[str, Any]]:
"""List Slack channels the bot has joined across all workspaces.
def _slack_has_raw_name(entry: Dict[str, Any]) -> bool:
return entry.get("name", "").startswith(_SLACK_RAW_ID_PREFIXES)
``users.conversations`` per workspace client (public + private member channels),
then merge DMs from session history. Missing channels:read falls back to
session history quietly rather than warning on every refresh.
"""
async def _slack_team_channels(team_id: str, client, seen_ids: set) -> List[Dict[str, Any]]:
"""``users.conversations`` for one workspace (public + private member channels), paginated."""
channels: List[Dict[str, Any]] = []
try:
cursor: Optional[str] = None
for _page in range(20): # safety cap on pagination
response = await client.users_conversations(
types="public_channel,private_channel",
exclude_archived=True,
limit=200,
cursor=cursor,
)
if not response.get("ok"):
error_code = response.get("error", "unknown")
_report_slack_failure(team_id, error_code, f"users.conversations not ok: {error_code}")
break
for ch in response.get("channels", []):
cid, name = ch.get("id"), ch.get("name")
if not cid or not name or cid in seen_ids:
continue
seen_ids.add(cid)
channels.append({"id": cid, "name": name, "type": "private" if ch.get("is_private") else "channel"})
cursor = (response.get("response_metadata") or {}).get("next_cursor")
if not cursor:
break
except Exception as e:
_report_slack_failure(team_id, _slack_api_error_code(e), str(e))
return channels
async def _slack_resolve_raw_names(client, channels: List[Dict[str, Any]]) -> None:
"""Name remaining raw-ID entries (DMs, channels outside bot scope) via
conversations.info + users.info once per base conversation, concurrently."""
unresolved_by_base: Dict[str, list] = {}
for entry in channels:
if _slack_has_raw_name(entry):
unresolved_by_base.setdefault(_slack_base_id(entry["id"]), []).append(entry)
if not unresolved_by_base:
return
async def _resolve_base(base_id: str, entries: list) -> None:
try:
resp = await client.conversations_info(channel=base_id)
if not resp.get("ok"):
return
ch_info = resp.get("channel", {})
resolved_name = resolved_type = None
if not ch_info.get("is_im"):
resolved_name = ch_info.get("name") or ch_info.get("name_normalized")
elif ch_info.get("user", ""):
user_resp = await client.users_info(user=ch_info["user"])
if user_resp.get("ok"):
u = user_resp["user"]
resolved_name = u.get("profile", {}).get("display_name") or u.get("real_name") or u.get("name")
resolved_type = "dm"
for entry in entries if resolved_name else ():
entry["name"] = resolved_name
if resolved_type:
entry["type"] = resolved_type
except Exception as e:
logger.debug("Channel directory: failed to resolve %s: %s", base_id, e)
await asyncio.gather(*[_resolve_base(bid, ents) for bid, ents in unresolved_by_base.items()])
async def _build_slack(adapter) -> List[Dict[str, Any]]:
"""List Slack channels the bot has joined across all workspaces, merged with
session-history DMs. Missing channels:read falls back to session history quietly."""
team_clients = getattr(adapter, "_team_clients", None) or {}
if not team_clients:
return await asyncio.to_thread(_build_from_sessions, "slack")
channels: List[Dict[str, Any]] = []
seen_ids: set = set()
for team_id, client in team_clients.items():
try:
cursor: Optional[str] = None
for _page in range(20): # safety cap on pagination
response = await client.users_conversations(
types="public_channel,private_channel",
exclude_archived=True,
limit=200,
cursor=cursor,
)
if not response.get("ok"):
error_code = response.get("error", "unknown")
_report_slack_failure(team_id, error_code, f"users.conversations not ok: {error_code}")
break
for ch in response.get("channels", []):
cid = ch.get("id")
name = ch.get("name")
if not cid or not name or cid in seen_ids:
continue
seen_ids.add(cid)
channels.append({"id": cid, "name": name, "type": "private" if ch.get("is_private") else "channel"})
cursor = (response.get("response_metadata") or {}).get("next_cursor")
if not cursor:
break
except Exception as e:
_report_slack_failure(team_id, _slack_api_error_code(e), str(e))
continue
channels.extend(await _slack_team_channels(team_id, client, seen_ids))
# Merge session-history DM/group entries, naming raw-ID entries from the
# API-discovered channels where the base conversation ID is known.
@@ -309,50 +334,12 @@ async def _build_slack(adapter) -> List[Dict[str, Any]]:
eid = entry.get("id")
if not isinstance(eid, str) or eid in seen_ids:
continue
if entry.get("name", "").startswith(_SLACK_RAW_ID_PREFIXES):
base_id = _slack_base_id(eid)
if base_id in api_name_lookup:
entry["name"] = api_name_lookup[base_id]
if _slack_has_raw_name(entry) and _slack_base_id(eid) in api_name_lookup:
entry["name"] = api_name_lookup[_slack_base_id(eid)]
channels.append(entry)
seen_ids.add(eid)
# Resolve remaining raw-ID entries (DMs, channels outside bot scope) via
# conversations.info + users.info once per base conversation, concurrently.
unresolved = [ch for ch in channels if ch.get("name", "").startswith(_SLACK_RAW_ID_PREFIXES)]
if unresolved:
client = next(iter(team_clients.values()))
unresolved_by_base: Dict[str, list] = {}
for entry in unresolved:
unresolved_by_base.setdefault(_slack_base_id(entry["id"]), []).append(entry)
async def _resolve_base(base_id: str, entries: list) -> None:
try:
resp = await client.conversations_info(channel=base_id)
if not resp.get("ok"):
return
ch_info = resp.get("channel", {})
resolved_name = None
resolved_type = None
if ch_info.get("is_im"):
peer_user = ch_info.get("user", "")
if peer_user:
user_resp = await client.users_info(user=peer_user)
if user_resp.get("ok"):
u = user_resp["user"]
resolved_name = u.get("profile", {}).get("display_name") or u.get("real_name") or u.get("name")
resolved_type = "dm"
else:
resolved_name = ch_info.get("name") or ch_info.get("name_normalized")
if resolved_name:
for entry in entries:
entry["name"] = resolved_name
if resolved_type:
entry["type"] = resolved_type
except Exception as e:
logger.debug("Channel directory: failed to resolve %s: %s", base_id, e)
await asyncio.gather(*[_resolve_base(bid, ents) for bid, ents in unresolved_by_base.items()])
await _slack_resolve_raw_names(next(iter(team_clients.values())), channels)
return channels
@@ -365,51 +352,49 @@ def _build_from_sessions(platform_name: str) -> List[Dict[str, str]]:
return _build_from_sessions_db(platform_name) or _build_from_sessions_json(platform_name)
def _extend_from_origins(entries: List[Dict[str, Any]], origins: Iterable[Tuple[Dict[str, Any], Any]]) -> None:
"""Append deduped directory entries for (origin, chat_type) pairs to *entries*.
def _entries_from_origins(platform_name: str, source: str, origins_fn) -> List[Dict[str, Any]]:
"""Deduped directory entries for the (origin, chat_type) pairs yielded by ``origins_fn()``.
Appends incrementally so a mid-iteration failure keeps the entries read so far.
"""
seen_ids = set()
for origin, chat_type in origins:
entry_id = _session_entry_id(origin)
if not entry_id or entry_id in seen_ids:
continue
seen_ids.add(entry_id)
entries.append({
"id": entry_id, "name": _session_entry_name(origin),
"type": chat_type, "thread_id": origin.get("thread_id"),
})
entries: List[Dict[str, Any]] = []
try:
seen_ids = set()
for origin, chat_type in origins_fn():
entry_id = _session_entry_id(origin)
if not entry_id or entry_id in seen_ids:
continue
seen_ids.add(entry_id)
entries.append({
"id": entry_id, "name": _session_entry_name(origin),
"type": chat_type, "thread_id": origin.get("thread_id"),
})
except Exception as e:
logger.debug("Channel directory: %s for %s: %s", source, platform_name, e)
return entries
def _build_from_sessions_db(platform_name: str) -> List[Dict[str, str]]:
"""Pull channels/contacts from state.db gateway session rows."""
entries: List[Dict[str, Any]] = []
try:
def _origins() -> Iterable[Tuple[Dict[str, Any], Any]]:
from hermes_state import get_shared_session_db, release_or_close
db = get_shared_session_db()
try:
lister = getattr(db, "list_gateway_sessions", None)
if not callable(lister):
return []
return
rows = lister(platform=platform_name, active_only=False)
finally:
release_or_close(db)
for row in rows:
origin = None
with contextlib.suppress(TypeError, ValueError):
origin = json.loads(row["origin_json"]) if row.get("origin_json") else None
if not isinstance(origin, dict) or not origin:
origin = {"chat_id": row.get("chat_id"), "thread_id": row.get("thread_id"), "chat_name": row.get("display_name")}
yield origin, row.get("chat_type") or "dm"
def _origin(row) -> Dict[str, Any]:
if row.get("origin_json"):
try:
parsed = json.loads(row["origin_json"])
if isinstance(parsed, dict) and parsed:
return parsed
except (TypeError, ValueError):
pass
return {"chat_id": row.get("chat_id"), "thread_id": row.get("thread_id"), "chat_name": row.get("display_name")}
_extend_from_origins(entries, ((_origin(row), row.get("chat_type") or "dm") for row in rows))
except Exception as e:
logger.debug("Channel directory: state.db session read failed for %s: %s", platform_name, e)
return entries
return _entries_from_origins(platform_name, "state.db session read failed", _origins)
def _build_from_sessions_json(platform_name: str) -> List[Dict[str, str]]:
@@ -417,25 +402,19 @@ def _build_from_sessions_json(platform_name: str) -> List[Dict[str, str]]:
sessions_path = get_hermes_home() / "sessions" / "sessions.json"
if not sessions_path.exists():
return []
entries: List[Dict[str, Any]] = []
try:
def _origins() -> Iterable[Tuple[Dict[str, Any], Any]]:
with open(sessions_path, encoding="utf-8") as f:
data = json.load(f)
for _key, session in data.items():
# Keys starting with "_" (e.g. the gateway's "_README") are metadata sentinels.
if str(_key).startswith("_") or not isinstance(session, dict):
continue
origin = session.get("origin") or {}
if origin.get("platform") == platform_name:
yield origin, session.get("chat_type", "dm")
def _origins():
for _key, session in data.items():
# Keys starting with "_" (e.g. the gateway's "_README") are
# metadata sentinels, not sessions.
if str(_key).startswith("_") or not isinstance(session, dict):
continue
origin = session.get("origin") or {}
if origin.get("platform") == platform_name:
yield origin, session.get("chat_type", "dm")
_extend_from_origins(entries, _origins())
except Exception as e:
logger.debug("Channel directory: failed to read sessions for %s: %s", platform_name, e)
return entries
return _entries_from_origins(platform_name, "failed to read sessions", _origins)
# --- Read / resolve --------------------------------------------------------
@@ -444,21 +423,19 @@ def load_directory() -> Dict[str, Any]:
"""Load the cached directory from disk, with aliases re-applied on read."""
directory_path = _directory_path()
if directory_path.exists():
try:
with contextlib.suppress(Exception):
with open(directory_path, encoding="utf-8") as f:
data = json.load(f)
# Aliases apply on read too, so new names take effect between timed rebuilds.
_apply_channel_aliases(data.setdefault("platforms", {}))
return data
except Exception:
pass
base = {"updated_at": None, "platforms": {}}
_apply_channel_aliases(base["platforms"])
return base
def lookup_channel_type(platform_name: str, chat_id: str) -> Optional[str]:
"""Return the channel ``type`` string (e.g. ``"channel"``, ``"forum"``) for *chat_id*, or *None* if unknown."""
"""Channel ``type`` string (e.g. ``"channel"``, ``"forum"``) for *chat_id*, or None if unknown."""
for ch in load_directory().get("platforms", {}).get(platform_name, []):
if ch.get("id") == chat_id:
return ch.get("type")
@@ -487,9 +464,7 @@ def resolve_channel_name(platform_name: str, name: str) -> Optional[str]:
# 1. Exact name match, including the display labels shown by send_message(action="list")
for ch in channels:
if _normalize_channel_query(ch["name"]) == query:
return ch["id"]
if _normalize_channel_query(_channel_target_name(platform_name, ch)) == query:
if query in (_normalize_channel_query(ch["name"]), _normalize_channel_query(_channel_target_name(platform_name, ch))):
return ch["id"]
# 2. Guild-qualified match for Discord ("GuildName/channel")
@@ -531,22 +506,17 @@ def format_directory_for_display(platforms: Optional[Dict[str, Any]] = None) ->
f"{plat_name}:<chat_id>, or bare '{plat_name}' for the home channel)"
)
elif plat_name == "discord":
# Group Discord channels by guild; DMs last.
# Group Discord channels by guild (sorted by name); DMs last, in discovery order.
guilds: Dict[str, List] = {}
dms: List = []
for ch in channels:
guild = ch.get("guild")
(guilds.setdefault(guild, []) if guild else dms).append(ch)
for guild_name, guild_channels in sorted(guilds.items()):
lines.append(f"Discord ({guild_name}):")
lines.extend(
f" discord:{_channel_target_name(plat_name, ch)}"
for ch in sorted(guild_channels, key=lambda c: c["name"])
)
(guilds.setdefault(ch["guild"], []) if ch.get("guild") else dms).append(ch)
groups = [(f"Discord ({g}):", sorted(chs, key=lambda c: c["name"])) for g, chs in sorted(guilds.items())]
if dms:
lines.append("Discord (DMs):")
lines.extend(f" discord:{_channel_target_name(plat_name, ch)}" for ch in dms)
groups.append(("Discord (DMs):", dms))
for header, group in groups:
lines.append(header)
lines.extend(f" discord:{_channel_target_name(plat_name, ch)}" for ch in group)
else:
lines.append(f"{plat_name.title()}:")
lines.extend(f" {plat_name}:{_channel_target_name(plat_name, ch)}" for ch in channels)
gateway/code_skew.py
+7 -16
View File
@@ -1,15 +1,11 @@
"""Detect when the gateway is running stale code after a hot ``git pull``.
The gateway's ``sys.modules`` is frozen at boot. If the checkout is updated
underneath it (manual ``git pull``, or the window before ``hermes update``'s
graceful restart), a first-time lazy import can resolve a freshly-pulled module
against a stale cached dependency -> ImportError
(``tests/test_stale_utils_module_import.py``). We snapshot the revision at
underneath it, a first-time lazy import can resolve a freshly-pulled module
against a stale cached dependency -> ImportError. We snapshot the revision at
startup so risky callers (e.g. ``/model`` switching) can refuse with a clear
"restart the gateway" message instead.
If the revision can't be read (non-git install, IO error) the boot snapshot
stays ``None`` and detection no-ops — never a false positive.
"restart the gateway" message. If the revision can't be read (non-git install,
IO error) the boot snapshot stays ``None`` and detection no-ops — never a false positive.
"""
from __future__ import annotations
@@ -41,17 +37,12 @@ def record_boot_fingerprint() -> None:
def _short(fingerprint: str) -> str:
"""Render a ``git:<ref>:<sha>`` fingerprint as a compact label."""
sha = fingerprint.rsplit(":", 1)[-1]
if sha and sha != "unresolved" and len(sha) > 10:
return sha[:10]
return sha or fingerprint
return sha[:10] if sha and sha != "unresolved" and len(sha) > 10 else (sha or fingerprint)
def detect_code_skew() -> tuple[str, str] | None:
"""Return ``(boot_rev, disk_rev)`` short labels if the checkout drifted
since boot, else ``None``."""
if _boot_fingerprint is None:
return None
current = _fingerprint()
"""``(boot_rev, disk_rev)`` short labels if the checkout drifted since boot, else ``None``."""
current = _fingerprint() if _boot_fingerprint is not None else None
if current is None or current == _boot_fingerprint:
return None
return _short(_boot_fingerprint), _short(current)
gateway/config.py
+268 -695
View File
File diff suppressed because it is too large Load Diff
gateway/config_env.py
+174 -241
View File
@@ -2,9 +2,9 @@
Runs after ``GatewayConfig.from_dict`` so env always wins over config.yaml /
gateway.json. Most platforms follow one shape — "credentials present in env
⇒ enable the platform and copy the values into ``extra``" — and are declared
as ``_Cred`` rows in ``_ENV_STEPS`` (source order = application order).
Platforms with unique gating live as small functions in the same table.
⇒ enable the platform and copy the values into ``extra``" — declared as
``_Cred`` rows in ``_ENV_STEPS`` (source order = application order).
Platforms with unique gating are small functions in the same table.
"""
import contextlib
@@ -13,7 +13,7 @@ import logging
import os
from dataclasses import dataclass
from functools import partial
from typing import Any, Callable, Dict, List, Optional
from typing import Any, Callable, Dict, Optional
from gateway.config import (
GatewayConfig,
@@ -30,15 +30,13 @@ logger = logging.getLogger("gateway.config")
getenv = _getenv_str
# Platforms for which the "explicitly disabled in config.yaml, but credentials
# are present in the environment" WARNING has already been emitted in this
# process. Config reloads on every turn, so the notice is one-time per platform
# per process — loud once at startup, never a per-turn drumbeat.
# Platforms already warned about "explicitly disabled in config.yaml, but
# credentials present in env". Config reloads every turn, so the notice is
# one-time per platform per process.
_EXPLICIT_DISABLE_WARNED: set = set()
# Env var(s) whose presence drives each platform's env-enable branch, named in
# the explicit-disable WARNING below.
# the explicit-disable WARNING.
_ENV_ENABLE_CREDENTIALS: dict = {
Platform.TELEGRAM: ("TELEGRAM_BOT_TOKEN",),
Platform.DISCORD: ("DISCORD_BOT_TOKEN",),
@@ -63,13 +61,8 @@ _ENV_ENABLE_CREDENTIALS: dict = {
def _warn_explicit_disable_beats_env(platform: Platform) -> None:
"""One-time WARNING: ``platforms.<x>.enabled: false`` wins over env creds.
Until #48820 credential presence force-enabled twelve platforms regardless
of an explicit ``enabled: false``; users relying on "creds in .env =
platform on" need to be told why it went dark. Names the platform, the
winning config key, and the env var(s) that used to override it.
"""
"""One-time WARNING: ``platforms.<x>.enabled: false`` wins over env creds (which
used to force-enable), so users relying on "creds in .env = on" learn why it went dark."""
if platform in _EXPLICIT_DISABLE_WARNED:
return
_EXPLICIT_DISABLE_WARNED.add(platform)
@@ -88,12 +81,12 @@ def _warn_explicit_disable_beats_env(platform: Platform) -> None:
# --- small value parsers -----------------------------------------------------
def _csv_list(value: str) -> List[str]:
def _csv_list(value: str) -> list:
return [part.strip() for part in value.split(",") if part.strip()]
def _int_or(default: int) -> Callable[[str], int]:
"""``int(raw.strip(), 10)`` with *default* on malformed/blank input (``_getenv_int`` semantics)."""
"""``int(raw.strip(), 10)`` with *default* on malformed/blank input."""
def parse(raw: str) -> int:
try:
return int(str(raw).strip(), 10)
@@ -106,6 +99,10 @@ def _strip_slash(value: str) -> str:
return value.rstrip("/")
def _strip_lower(value: str) -> str:
return value.strip().lower()
def _truthy_token(value: str) -> bool:
return value.lower() in {"true", "1", "yes", "on"}
@@ -121,11 +118,7 @@ def _env_first(envs) -> str:
"""First truthy value among *envs* (a single name or a tuple of alternatives)."""
if isinstance(envs, str):
return getenv(envs)
for env in envs:
value = getenv(env)
if value:
return value
return ""
return next((v for v in map(getenv, envs) if v), "")
# --- reusable steps -----------------------------------------------------------
@@ -136,8 +129,8 @@ _INT = object() # spec marker: ``int(value)``, silently skipped when malformed
def _env_extras(extra: Dict[str, Any], spec, *, strip: bool = False) -> None:
"""``extra[key] = fn(value)`` for each ``(key, env[, fn])`` whose env value is truthy.
With ``strip=True`` the value is stripped BEFORE the truthiness check. ``fn=_INT`` parses an
int and silently ignores a non-integer value.
``strip=True`` strips BEFORE the truthiness check. ``fn=_INT`` parses an int
and silently ignores a non-integer value.
"""
for key, env, *fn in spec:
value = getenv(env)
@@ -159,10 +152,8 @@ def _env_home_channel(config: GatewayConfig, platform: Platform, env_base: str,
chat_id = chat_id.strip()
if chat_id and platform in config.platforms:
config.platforms[platform].home_channel = HomeChannel(
platform=platform,
chat_id=chat_id,
name=getenv(f"{env_base}_NAME", "Home"),
thread_id=getenv(f"{env_base}_THREAD_ID") or None,
platform=platform, chat_id=chat_id,
name=getenv(f"{env_base}_NAME", "Home"), thread_id=getenv(f"{env_base}_THREAD_ID") or None,
)
@@ -173,18 +164,14 @@ def _env_reply_mode(config: GatewayConfig, platform: Platform, env: str) -> None
def _enable_from_env(config: GatewayConfig, platform: Platform) -> PlatformConfig:
"""Enable *platform* because its env credentials are present — unless config.yaml explicitly disabled it.
READS (does not pop) the ``_enabled_explicit`` marker: the registry-driven plugin-enable pass later
also needs it. The marker is scrubbed for all platforms at the end of ``_apply_env_overrides``.
"""
"""Enable *platform* on env credentials unless config.yaml explicitly disabled it.
READS (does not pop) ``_enabled_explicit``: the plugin-enable pass also needs it."""
if platform not in config.platforms:
config.platforms[platform] = PlatformConfig(enabled=True)
return config.platforms[platform]
platform_config = config.platforms[platform]
if not platform_config.enabled:
if platform_config.extra.get("_enabled_explicit", False):
# Credentials are present (that is why we are here) but the user said no in config.yaml.
_warn_explicit_disable_beats_env(platform)
else:
platform_config.enabled = True
@@ -192,12 +179,11 @@ def _enable_from_env(config: GatewayConfig, platform: Platform) -> PlatformConfi
def _enable_port_bound_from_env(config: GatewayConfig, platform: Platform) -> PlatformConfig:
"""Enable a port-binding platform (api_server/webhook) unless config.yaml explicitly disabled it.
"""Enable a port-binding platform unless config.yaml explicitly disabled it.
In multiplex mode a secondary profile pins ``platforms.<x>.enabled: false`` so it shares the
default profile's listener instead of binding its own port, yet still inherits the process env;
without this guard the env presence would force-enable the listener and trip MultiplexConfigError.
POPs the ``_enabled_explicit`` marker: these branches are terminal (no later registry pass).
A multiplex secondary profile pins ``enabled: false`` to share the default profile's
listener yet inherits the process env; without this guard env presence would
force-enable it and trip MultiplexConfigError. POPs the marker (terminal branch).
"""
platform_config = config.platforms.setdefault(platform, PlatformConfig())
explicit = platform_config.extra.pop("_enabled_explicit", False)
@@ -210,12 +196,12 @@ def _enable_port_bound_from_env(config: GatewayConfig, platform: Platform) -> Pl
class _Cred:
"""Credential-gated platform enable, applied as ``step(config)``.
``creds``: env names that must ALL be truthy; an inner tuple lists alternatives (ANY).
``token``: env whose truthy value becomes ``PlatformConfig.token``.
``fixed``: ``(extra_key, env[, default[, fn]])`` always written once enabled (``env`` may be a
tuple of alternatives). ``optional`` / ``optional_stripped``: ``_env_extras`` specs, written only
when truthy. ``warn_missing``: ``(env, msg)`` logged BEFORE enabling when *env* is blank. ``then``: unique tail ``fn(config, platform_config)``.
``home``: ``_env_home_channel`` env base applied only when the gate passed.
``creds``: env names that must ALL be truthy (inner tuple = ANY of). ``token``: env
stored as ``PlatformConfig.token`` even when yaml disables the adapter (sending skills
use it). ``fixed``: ``(extra_key, env[, default[, fn]])`` always written once enabled.
``optional*``: ``_env_extras`` specs. ``warn_missing``: ``(env, msg)`` logged BEFORE
enabling when blank. ``then``: tail ``fn(config, platform_config)``. ``home``:
``_env_home_channel`` env base applied only when the gate passed.
"""
platform: Platform
creds: tuple
@@ -268,14 +254,12 @@ def _telegram_fallback_ips(config: GatewayConfig) -> None:
def _whatsapp(config: GatewayConfig) -> None:
"""WhatsApp (Baileys bridge) uses a flag, not credentials."""
"""WhatsApp (Baileys bridge) uses a flag, not credentials; an explicit false overrides YAML."""
raw = getenv("WHATSAPP_ENABLED")
enabled = is_truthy_value(raw)
disabled_explicitly = raw.lower() in {"false", "0", "no"}
if Platform.WHATSAPP in config.platforms:
# YAML config exists — respect explicit disable; otherwise keep whatever the YAML set.
wa_cfg = config.platforms[Platform.WHATSAPP]
if disabled_explicitly:
wa_cfg = config.platforms.get(Platform.WHATSAPP)
if wa_cfg is not None:
if raw.lower() in {"false", "0", "no"}:
wa_cfg.enabled = False
elif enabled:
wa_cfg.enabled = True
@@ -283,34 +267,21 @@ def _whatsapp(config: GatewayConfig) -> None:
config.platforms[Platform.WHATSAPP] = PlatformConfig(enabled=True)
def _slack(config: GatewayConfig) -> None:
slack_token = getenv("SLACK_BOT_TOKEN")
if slack_token:
if Platform.SLACK not in config.platforms:
config.platforms[Platform.SLACK] = PlatformConfig(enabled=True)
else:
slack_config = config.platforms[Platform.SLACK]
if not slack_config.enabled and not slack_config.extra.get("_enabled_explicit", False):
# Top-level Slack settings such as channel prompts must not turn an env-token setup
# into a disabled platform; only an explicit enabled: false should.
slack_config.enabled = True
elif not slack_config.enabled:
_warn_explicit_disable_beats_env(Platform.SLACK)
# Token is stored even when yaml disables the adapter so Slack-sending skills can use it.
config.platforms[Platform.SLACK].token = slack_token
def _slack_home(config: GatewayConfig) -> None:
"""SLACK_HOME_CHANNEL creates a disabled Slack entry if needed and keeps
user_id/scope_id provenance when the chat_id is unchanged."""
slack_home = getenv("SLACK_HOME_CHANNEL")
if slack_home:
slack_config = config.platforms.setdefault(Platform.SLACK, PlatformConfig(enabled=False))
existing_home = slack_config.home_channel
same_home = existing_home is not None and existing_home.chat_id == slack_home
slack_config.home_channel = HomeChannel(
platform=Platform.SLACK,
chat_id=slack_home,
name=getenv("SLACK_HOME_CHANNEL_NAME"),
thread_id=getenv("SLACK_HOME_CHANNEL_THREAD_ID") or None,
user_id=existing_home.user_id if same_home else None,
scope_id=existing_home.scope_id if same_home else None,
)
if not slack_home:
return
slack_config = config.platforms.setdefault(Platform.SLACK, PlatformConfig(enabled=False))
existing_home = slack_config.home_channel
same_home = existing_home is not None and existing_home.chat_id == slack_home
slack_config.home_channel = HomeChannel(
platform=Platform.SLACK, chat_id=slack_home,
name=getenv("SLACK_HOME_CHANNEL_NAME"), thread_id=getenv("SLACK_HOME_CHANNEL_THREAD_ID") or None,
user_id=existing_home.user_id if same_home else None,
scope_id=existing_home.scope_id if same_home else None,
)
def _matrix_e2ee(config: GatewayConfig, matrix_config: PlatformConfig) -> None:
@@ -324,15 +295,12 @@ def _matrix_e2ee(config: GatewayConfig, matrix_config: PlatformConfig) -> None:
_env_extras(matrix_config.extra, (("device_id", "MATRIX_DEVICE_ID"),))
def _sms(config: GatewayConfig) -> None:
if getenv("TWILIO_ACCOUNT_SID"):
_enable_from_env(config, Platform.SMS).api_key = getenv("TWILIO_AUTH_TOKEN")
def _sms_api_key(config: GatewayConfig, sms_config: PlatformConfig) -> None:
sms_config.api_key = getenv("TWILIO_AUTH_TOKEN")
def _api_server(config: GatewayConfig) -> None:
"""Require a usable key: API_SERVER_ENABLED alone would load an unauthenticated platform whose
adapter refuses to start at connect() anyway, leaving the reconnect watcher spinning forever.
Same strength bar as the startup guard (has_usable_secret, min_length=16)."""
"""Require a usable key: an unauthenticated adapter refuses to start and the reconnect watcher would spin."""
key = getenv("API_SERVER_KEY")
if not _has_usable_api_server_key(key):
return
@@ -355,18 +323,11 @@ def _msgraph_webhook(config: GatewayConfig) -> None:
client_state = getenv("MSGRAPH_WEBHOOK_CLIENT_STATE")
resources = getenv("MSGRAPH_WEBHOOK_ACCEPTED_RESOURCES")
allowed_cidrs = getenv("MSGRAPH_WEBHOOK_ALLOWED_SOURCE_CIDRS")
if not (
enabled
or Platform.MSGRAPH_WEBHOOK in config.platforms
or getenv("MSGRAPH_WEBHOOK_PORT")
or client_state
or resources
or allowed_cidrs
):
if not (enabled or Platform.MSGRAPH_WEBHOOK in config.platforms or getenv("MSGRAPH_WEBHOOK_PORT")
or client_state or resources or allowed_cidrs):
return
msgraph_cfg = config.platforms.setdefault(Platform.MSGRAPH_WEBHOOK, PlatformConfig())
# Same explicit-disable guard as the webhook branch, but READ (don't pop) the marker:
# the relay-exclusive pass below still consults it; the end-of-function scrub removes it.
# Same guard as webhook, but READ (don't pop) the marker: the relay pass still consults it.
if enabled and (not msgraph_cfg.extra.get("_enabled_explicit", False) or msgraph_cfg.enabled):
msgraph_cfg.enabled = True
_env_extras(msgraph_cfg.extra, (("port", "MSGRAPH_WEBHOOK_PORT", _INT),))
@@ -383,9 +344,8 @@ def _qq_home(config: GatewayConfig, qq_config: PlatformConfig) -> None:
name_env = "QQBOT_HOME_CHANNEL_NAME"
if not qq_home:
# Back-compat: accept the pre-rename name and log a one-time warning.
legacy_home = getenv("QQ_HOME_CHANNEL").strip()
if legacy_home:
qq_home = legacy_home
qq_home = getenv("QQ_HOME_CHANNEL").strip()
if qq_home:
name_env = "QQ_HOME_CHANNEL_NAME"
logger.warning(
"QQ_HOME_CHANNEL is deprecated; rename to QQBOT_HOME_CHANNEL "
@@ -393,8 +353,7 @@ def _qq_home(config: GatewayConfig, qq_config: PlatformConfig) -> None:
)
if qq_home:
qq_config.home_channel = HomeChannel(
platform=Platform.QQBOT,
chat_id=qq_home,
platform=Platform.QQBOT, chat_id=qq_home,
name=getenv("QQBOT_HOME_CHANNEL_NAME") or getenv(name_env, "Home"),
thread_id=getenv("QQBOT_HOME_CHANNEL_THREAD_ID") or getenv("QQ_HOME_CHANNEL_THREAD_ID") or None,
)
@@ -408,114 +367,106 @@ def _session_settings(config: GatewayConfig) -> None:
setattr(config.default_reset_policy, attr, int(raw))
def _enable_plugin_platforms_from_env(config: GatewayConfig) -> None:
"""Registry-driven enable for plugin platforms (built-ins have explicit rows in ``_ENV_STEPS``).
def _plugin_probe_seed(entry) -> Optional[dict]:
"""``env_enablement_fn()`` result as a non-empty dict, else None."""
if entry.env_enablement_fn is None:
return None
try:
seed = entry.env_enablement_fn()
except Exception as e:
logger.debug("env_enablement_fn for %s raised: %s", entry.name, e)
return None
return seed if isinstance(seed, dict) and seed else None
A plugin platform is enabled when its credentials are configured (``is_connected``) and its deps
are present (passive ``check_fn``) or installable on demand (``ensure_deps_fn`` — run later by
``create_adapter()``, never here: the active installer used to be wired as ``check_fn`` and this
sweep pip-installed SDKs on every ``load_gateway_config()`` call, boot-looping the desktop app).
``is_connected`` MUST gate enablement: ``check_fn`` alone ("is the SDK importable?") would enable
platforms the user never configured and the gateway would retry-connect forever with no token.
def _plugin_is_configured(entry, existing_extra: dict, seed: Optional[dict]) -> bool:
"""``entry.is_connected`` on a transient ``enabled=True`` view seeded with env extras."""
try:
# Seed extras so ``is_connected`` implementations reading ``config.extra`` see
# post-enable state; never mutate the real config for the probe.
for k, v in (seed or {}).items():
if k != "home_channel":
existing_extra.setdefault(k, v)
configured = bool(entry.is_connected(PlatformConfig(enabled=True, extra=existing_extra)))
except Exception as exc:
logger.debug("is_connected for %s raised: %s — skipping enablement", entry.name, exc)
return False
if not configured:
logger.debug(
"Plugin platform '%s' available but not configured "
"(is_connected returned False) — skipping enable",
entry.name,
)
return configured
def _enable_plugin_platform(config: GatewayConfig, entry) -> None:
try:
platform = Platform(entry.name)
except Exception as e:
logger.debug("unknown platform name %r: %s", entry.name, e)
return
existing_cfg = config.platforms.get(platform)
already_enabled = existing_cfg is not None and existing_cfg.enabled
existing_extra = dict(existing_cfg.extra or {}) if existing_cfg is not None else {}
# Never re-enable a platform the user explicitly disabled (marker set by the YAML loader).
if existing_cfg is not None and not already_enabled and existing_extra.get("_enabled_explicit", False):
return
seed = _plugin_probe_seed(entry)
# Only consult is_connected for platforms not already enabled by YAML/env.
if not already_enabled and entry.is_connected is not None and not _plugin_is_configured(entry, existing_extra, seed):
return
# Verify dependencies LAST — only for platforms already enabled or past the credential gate.
try:
deps_ok = bool(entry.check_fn())
except Exception as e:
logger.debug("check_fn for %s raised: %s", entry.name, e)
deps_ok = False
if not deps_ok and entry.ensure_deps_fn is None:
return
platform_config = config.platforms.setdefault(platform, PlatformConfig())
platform_config.enabled = True
if seed:
# Commit the env-seeded extras (reuse the probe result; don't call env_enablement_fn twice).
seed = dict(seed)
home = seed.pop("home_channel", None)
platform_config.extra.update(seed)
if isinstance(home, dict) and home.get("chat_id"):
platform_config.home_channel = HomeChannel(
platform=platform, chat_id=str(home["chat_id"]), name=str(home.get("name") or "Home"),
thread_id=str(home["thread_id"]) if home.get("thread_id") else None,
)
def _enable_plugin_platforms_from_env(config: GatewayConfig) -> None:
"""Registry-driven enable for plugin platforms (built-ins have rows in ``_ENV_STEPS``).
Enabled when credentials are configured (``is_connected`` MUST gate: ``check_fn``
alone would enable unconfigured platforms that retry-connect forever) and deps are
present (``check_fn``) or installable later by ``create_adapter()`` — never here:
installing in this sweep pip-installed SDKs on every load and boot-looped the app.
"""
try:
from hermes_cli.plugins import discover_plugins
discover_plugins() # idempotent
from gateway.platform_registry import platform_registry
for entry in platform_registry.plugin_entries():
try:
platform = Platform(entry.name)
except Exception as e:
logger.debug("unknown platform name %r: %s", entry.name, e)
continue
existing_cfg = config.platforms.get(platform)
# Never re-enable a platform the user explicitly disabled (marker set by the YAML loader).
if (
existing_cfg is not None
and not existing_cfg.enabled
and bool((existing_cfg.extra or {}).get("_enabled_explicit", False))
):
continue
# Seed candidate extras so plugins whose ``is_connected`` reads ``config.extra`` (Google Chat)
# see the same state they will after enablement.
seed_for_probe = None
if entry.env_enablement_fn is not None:
try:
seed_for_probe = entry.env_enablement_fn()
except Exception as e:
logger.debug("env_enablement_fn for %s raised: %s", entry.name, e)
seed_for_probe = None
has_seed = isinstance(seed_for_probe, dict) and bool(seed_for_probe)
# Only consult is_connected for platforms not already enabled by YAML/env (keep that decision).
if (existing_cfg is None or not existing_cfg.enabled) and entry.is_connected is not None:
try:
# Probe with ``enabled=True``: we ask "would this be configured if enabled?" —
# some ``is_connected`` short-circuit on ``config.enabled`` being False.
probe_cfg = existing_cfg
if probe_cfg is None or not probe_cfg.enabled:
probe_cfg = PlatformConfig(
enabled=True,
extra=dict(existing_cfg.extra or {}) if existing_cfg is not None else {},
)
if has_seed:
# Transient view; never mutate ``existing_cfg`` for the probe.
probe_extra = dict(probe_cfg.extra or {})
for k, v in seed_for_probe.items():
if k != "home_channel":
probe_extra.setdefault(k, v)
probe_cfg = PlatformConfig(enabled=True, extra=probe_extra)
configured = bool(entry.is_connected(probe_cfg))
except Exception as exc:
logger.debug("is_connected for %s raised: %s — skipping enablement", entry.name, exc)
configured = False
if not configured:
logger.debug(
"Plugin platform '%s' available but not configured "
"(is_connected returned False) — skipping enable",
entry.name,
)
continue
# Verify dependencies LAST — only for platforms already enabled or past the credential gate.
try:
deps_ok = bool(entry.check_fn())
except Exception as e:
logger.debug("check_fn for %s raised: %s", entry.name, e)
deps_ok = False
if not deps_ok and entry.ensure_deps_fn is None:
continue
platform_config = config.platforms.setdefault(platform, PlatformConfig())
platform_config.enabled = True
# Commit the env-seeded extras (reuse the probe result; don't call env_enablement_fn twice).
if has_seed:
seed = dict(seed_for_probe)
home = seed.pop("home_channel", None)
platform_config.extra.update(seed)
if isinstance(home, dict) and home.get("chat_id"):
platform_config.home_channel = HomeChannel(
platform=platform,
chat_id=str(home["chat_id"]),
name=str(home.get("name") or "Home"),
thread_id=str(home["thread_id"]) if home.get("thread_id") else None,
)
_enable_plugin_platform(config, entry)
except Exception as e:
logger.debug("Plugin platform enable pass failed: %s", e)
def _relay(config: GatewayConfig) -> None:
"""Relay (generic connector-fronted platform, EXPERIMENTAL). Enabled by GATEWAY_RELAY_URL (env)
or gateway.relay_url (config.yaml). The adapter dials OUT to the connector (no inbound port),
so it only needs Platform.RELAY present+enabled for start_gateway()'s connect loop. The
connected-checker keys on extra["relay_url"], so the URL is mirrored into extra here.
"""Relay (connector-fronted, EXPERIMENTAL): enabled by GATEWAY_RELAY_URL or
gateway.relay_url; the URL is mirrored into extra["relay_url"] for the connected-checker.
Relay-exclusive: a GATEWAY_RELAY_URL env stamp marks a connector-fronted deployment where the
connector owns every platform connection; a directly-connected adapter in the same process
would be a second unmanaged ingress (duplicate deliveries, split sessions, a live socket that
disarms scale-to-zero). So the env stamp disables all other messaging platforms — even ones
explicitly enabled in config.yaml. Non-messaging surfaces (local, api_server, webhook — same
exclusion set as the scale-to-zero arm gate) are untouched; relay via gateway.relay_url only
keeps the old additive behavior. Opt-out GATEWAY_RELAY_ALLOW_DIRECT_PLATFORMS=true is likewise
a deploy-stamp env var read through the profile-scope-aware getenv.
Relay-exclusive: the GATEWAY_RELAY_URL env stamp means the connector owns every
platform connection; a directly-connected adapter would be a second unmanaged ingress
(duplicate deliveries, split sessions, a socket disarming scale-to-zero). So the env
stamp disables all other messaging platforms — even ones explicitly enabled — except
non-messaging surfaces (local, api_server, webhook; same set as the scale-to-zero arm
gate). relay_url from YAML only keeps the additive behavior. Opt out with
GATEWAY_RELAY_ALLOW_DIRECT_PLATFORMS=true.
"""
relay_url_env = getenv("GATEWAY_RELAY_URL").strip()
existing_relay = config.platforms.get(Platform.RELAY)
@@ -554,10 +505,9 @@ def _scrub_explicit_markers(config: GatewayConfig) -> None:
platform_config.extra.pop("_enabled_explicit", None)
# Application order is significant: e.g. Telegram's reply mode may create the (disabled) platform
# entry BEFORE its home channel is read, while Discord reads home first; a home channel is only
# attached to a platform that already exists. Relay-exclusive disabling runs after the plugin
# pass and before the marker scrub, which must be last.
# Order is significant: a home channel only attaches to a platform that already exists
# (Telegram's reply mode may create the entry first; Discord reads home first). Relay
# disabling runs after the plugin pass; the marker scrub must be last.
_ENV_STEPS: tuple = (
_Cred(Platform.TELEGRAM, ("TELEGRAM_BOT_TOKEN",), token="TELEGRAM_BOT_TOKEN"),
_ReplyMode(Platform.TELEGRAM, "TELEGRAM_REPLY_TO_MODE"),
@@ -568,29 +518,25 @@ _ENV_STEPS: tuple = (
_ReplyMode(Platform.DISCORD, "DISCORD_REPLY_TO_MODE"),
_whatsapp,
_Home(Platform.WHATSAPP, "WHATSAPP_HOME_CHANNEL"),
# WhatsApp Cloud API (official Business Platform via Meta). Distinct from the Baileys bridge;
# WhatsApp Cloud API (Meta Business Platform). Distinct from the Baileys bridge;
# both adapters can run in parallel against different phone numbers.
_Cred(
Platform.WHATSAPP_CLOUD, ("WHATSAPP_CLOUD_PHONE_NUMBER_ID", "WHATSAPP_CLOUD_ACCESS_TOKEN"),
fixed=(("phone_number_id", "WHATSAPP_CLOUD_PHONE_NUMBER_ID"), ("access_token", "WHATSAPP_CLOUD_ACCESS_TOKEN")),
optional=(
("app_id", "WHATSAPP_CLOUD_APP_ID"),
("app_secret", "WHATSAPP_CLOUD_APP_SECRET"),
("waba_id", "WHATSAPP_CLOUD_WABA_ID"),
("verify_token", "WHATSAPP_CLOUD_VERIFY_TOKEN"), # Meta hub.verify_token shared secret
("webhook_host", "WHATSAPP_CLOUD_WEBHOOK_HOST"),
("webhook_port", "WHATSAPP_CLOUD_WEBHOOK_PORT", _INT),
("webhook_path", "WHATSAPP_CLOUD_WEBHOOK_PATH"),
("api_version", "WHATSAPP_CLOUD_API_VERSION"),
("app_id", "WHATSAPP_CLOUD_APP_ID"), ("app_secret", "WHATSAPP_CLOUD_APP_SECRET"),
("waba_id", "WHATSAPP_CLOUD_WABA_ID"), ("verify_token", "WHATSAPP_CLOUD_VERIFY_TOKEN"),
("webhook_host", "WHATSAPP_CLOUD_WEBHOOK_HOST"), ("webhook_port", "WHATSAPP_CLOUD_WEBHOOK_PORT", _INT),
("webhook_path", "WHATSAPP_CLOUD_WEBHOOK_PATH"), ("api_version", "WHATSAPP_CLOUD_API_VERSION"),
),
),
_Home(Platform.WHATSAPP_CLOUD, "WHATSAPP_CLOUD_HOME_CHANNEL"),
_slack,
_Cred(Platform.SLACK, ("SLACK_BOT_TOKEN",), token="SLACK_BOT_TOKEN"),
_slack_home,
_Cred(
Platform.SIGNAL, ("SIGNAL_HTTP_URL", "SIGNAL_ACCOUNT"),
fixed=(
("http_url", "SIGNAL_HTTP_URL"),
("account", "SIGNAL_ACCOUNT"),
("http_url", "SIGNAL_HTTP_URL"), ("account", "SIGNAL_ACCOUNT"),
("ignore_stories", "SIGNAL_IGNORE_STORIES", "true", is_truthy_value),
),
),
@@ -615,7 +561,7 @@ _ENV_STEPS: tuple = (
fixed=(("address", "EMAIL_ADDRESS"), ("imap_host", "EMAIL_IMAP_HOST"), ("smtp_host", "EMAIL_SMTP_HOST")),
),
_Home(Platform.EMAIL, "EMAIL_HOME_ADDRESS"),
_sms,
_Cred(Platform.SMS, ("TWILIO_ACCOUNT_SID",), then=_sms_api_key),
_Home(Platform.SMS, "SMS_HOME_CHANNEL"),
_api_server,
_webhook,
@@ -628,10 +574,8 @@ _ENV_STEPS: tuple = (
_Cred(
Platform.FEISHU, ("FEISHU_APP_ID", "FEISHU_APP_SECRET"),
fixed=(
("app_id", "FEISHU_APP_ID"),
("app_secret", "FEISHU_APP_SECRET"),
("domain", "FEISHU_DOMAIN", "feishu"),
("connection_mode", "FEISHU_CONNECTION_MODE", "websocket"),
("app_id", "FEISHU_APP_ID"), ("app_secret", "FEISHU_APP_SECRET"),
("domain", "FEISHU_DOMAIN", "feishu"), ("connection_mode", "FEISHU_CONNECTION_MODE", "websocket"),
),
optional=(("encrypt_key", "FEISHU_ENCRYPT_KEY"), ("verification_token", "FEISHU_VERIFICATION_TOKEN")),
home="FEISHU_HOME_CHANNEL",
@@ -645,15 +589,12 @@ _ENV_STEPS: tuple = (
_Cred(
Platform.WECOM_CALLBACK, ("WECOM_CALLBACK_CORP_ID", "WECOM_CALLBACK_CORP_SECRET"),
fixed=(
("corp_id", "WECOM_CALLBACK_CORP_ID"),
("corp_secret", "WECOM_CALLBACK_CORP_SECRET"),
("agent_id", "WECOM_CALLBACK_AGENT_ID"),
("token", "WECOM_CALLBACK_TOKEN"),
("corp_id", "WECOM_CALLBACK_CORP_ID"), ("corp_secret", "WECOM_CALLBACK_CORP_SECRET"),
("agent_id", "WECOM_CALLBACK_AGENT_ID"), ("token", "WECOM_CALLBACK_TOKEN"),
("encoding_aes_key", "WECOM_CALLBACK_ENCODING_AES_KEY"),
# No default: an unset WECOM_CALLBACK_HOST leaves extra.host falsy so the adapter's
# dual-stack DEFAULT_HOST=None applies (binds IPv4 + IPv6; "0.0.0.0" was IPv4-only).
("host", "WECOM_CALLBACK_HOST"),
("port", "WECOM_CALLBACK_PORT", "", _int_or(8645)),
# No host default: a falsy extra.host lets the adapter's dual-stack DEFAULT_HOST=None
# apply (binds IPv4 + IPv6; "0.0.0.0" was IPv4-only).
("host", "WECOM_CALLBACK_HOST"), ("port", "WECOM_CALLBACK_PORT", "", _int_or(8645)),
),
),
# Weixin (personal WeChat via iLink Bot API)
@@ -661,12 +602,9 @@ _ENV_STEPS: tuple = (
Platform.WEIXIN, (("WEIXIN_TOKEN", "WEIXIN_ACCOUNT_ID"),), token="WEIXIN_TOKEN",
optional=(("account_id", "WEIXIN_ACCOUNT_ID"),),
optional_stripped=(
("base_url", "WEIXIN_BASE_URL", _strip_slash),
("cdn_base_url", "WEIXIN_CDN_BASE_URL", _strip_slash),
("dm_policy", "WEIXIN_DM_POLICY", str.lower),
("group_policy", "WEIXIN_GROUP_POLICY", str.lower),
("allow_from", "WEIXIN_ALLOWED_USERS"),
("group_allow_from", "WEIXIN_GROUP_ALLOWED_USERS"),
("base_url", "WEIXIN_BASE_URL", _strip_slash), ("cdn_base_url", "WEIXIN_CDN_BASE_URL", _strip_slash),
("dm_policy", "WEIXIN_DM_POLICY", str.lower), ("group_policy", "WEIXIN_GROUP_POLICY", str.lower),
("allow_from", "WEIXIN_ALLOWED_USERS"), ("group_allow_from", "WEIXIN_GROUP_ALLOWED_USERS"),
("split_multiline_messages", "WEIXIN_SPLIT_MULTILINE_MESSAGES"),
),
home="WEIXIN_HOME_CHANNEL", home_strip=True,
@@ -675,8 +613,7 @@ _ENV_STEPS: tuple = (
_Cred(
Platform.BLUEBUBBLES, ("BLUEBUBBLES_SERVER_URL", "BLUEBUBBLES_PASSWORD"),
fixed=(
("server_url", "BLUEBUBBLES_SERVER_URL", "", _strip_slash),
("password", "BLUEBUBBLES_PASSWORD"),
("server_url", "BLUEBUBBLES_SERVER_URL", "", _strip_slash), ("password", "BLUEBUBBLES_PASSWORD"),
("webhook_host", "BLUEBUBBLES_WEBHOOK_HOST", "127.0.0.1"),
("webhook_port", "BLUEBUBBLES_WEBHOOK_PORT", "", _int_or(8645)),
("webhook_path", "BLUEBUBBLES_WEBHOOK_PATH", "/bluebubbles-webhook"),
@@ -698,14 +635,10 @@ _ENV_STEPS: tuple = (
Platform.YUANBAO, (("YUANBAO_APP_ID", "YUANBAO_APP_KEY"), "YUANBAO_APP_SECRET"),
fixed=(("app_id", ("YUANBAO_APP_ID", "YUANBAO_APP_KEY")), ("app_secret", "YUANBAO_APP_SECRET")),
optional=(
("bot_id", "YUANBAO_BOT_ID"),
("ws_url", "YUANBAO_WS_URL"),
("api_domain", "YUANBAO_API_DOMAIN"),
("route_env", "YUANBAO_ROUTE_ENV"),
("dm_policy", "YUANBAO_DM_POLICY", lambda v: v.strip().lower()),
("dm_allow_from", "YUANBAO_DM_ALLOW_FROM"),
("group_policy", "YUANBAO_GROUP_POLICY", lambda v: v.strip().lower()),
("group_allow_from", "YUANBAO_GROUP_ALLOW_FROM"),
("bot_id", "YUANBAO_BOT_ID"), ("ws_url", "YUANBAO_WS_URL"),
("api_domain", "YUANBAO_API_DOMAIN"), ("route_env", "YUANBAO_ROUTE_ENV"),
("dm_policy", "YUANBAO_DM_POLICY", _strip_lower), ("dm_allow_from", "YUANBAO_DM_ALLOW_FROM"),
("group_policy", "YUANBAO_GROUP_POLICY", _strip_lower), ("group_allow_from", "YUANBAO_GROUP_ALLOW_FROM"),
),
home="YUANBAO_HOME_CHANNEL",
),
gateway/config_loader.py
+52 -68
View File
@@ -62,31 +62,32 @@ def _is_dict(value: Any) -> bool:
return isinstance(value, dict)
def _dm_behavior_choice(value: Any, default: str = "pair") -> str:
return _normalize_choice(value, {"pair", "ignore"}, default)
def _presence(*keys: str) -> tuple:
return tuple((k, k, "presence", None, None) for k in keys)
# (yaml key, gw_data key, mode, accept(value) -> bool, transform(value))
_TOPLEVEL_BRIDGE: tuple = (
("session_reset", "default_reset_policy", "presence", lambda v: bool(v) and isinstance(v, dict), None),
("quick_commands", "quick_commands", "none", _quick_commands_ok, None),
("stt", "stt", "presence", _is_dict, None),
("stt_echo_transcripts", "stt_echo_transcripts", "presence", None, None),
("group_sessions_per_user", "group_sessions_per_user", "presence", None, None),
("thread_sessions_per_user", "thread_sessions_per_user", "presence", None, None),
*_presence("stt_echo_transcripts", "group_sessions_per_user", "thread_sessions_per_user"),
("multiplex_profiles", "multiplex_profiles", "gwdata", None, None),
("multiplex_profile_allowlist", "multiplex_profile_allowlist", "presence", None, None),
("room_link_url", "room_link_url", "presence", None, None),
*_presence("multiplex_profile_allowlist", "room_link_url"),
("profile_routes", "profile_routes", "none", lambda v: isinstance(v, list), None),
("max_concurrent_sessions", "max_concurrent_sessions", "presence", None, None),
*_presence("max_concurrent_sessions"),
("systemd_watchdog_seconds", "systemd_watchdog_seconds", "nested", None, None),
("streaming", "streaming", "dict", None, None),
("reset_triggers", "reset_triggers", "presence", None, None),
("always_log_local", "always_log_local", "presence", None, None),
("write_sessions_json", "write_sessions_json", "presence", None, None),
("loop_watchdog", "loop_watchdog", "presence", None, None),
("loop_watchdog_probe_interval_s", "loop_watchdog_probe_interval_s", "presence", None, None),
("loop_watchdog_probe_timeout_s", "loop_watchdog_probe_timeout_s", "presence", None, None),
("loop_watchdog_max_strikes", "loop_watchdog_max_strikes", "presence", None, None),
("filter_silence_narration", "filter_silence_narration", "presence", None, None),
("unauthorized_dm_behavior", "unauthorized_dm_behavior", "presence", None,
lambda v: _normalize_choice(v, {"pair", "ignore"}, "pair")),
*_presence(
"reset_triggers", "always_log_local", "write_sessions_json", "loop_watchdog",
"loop_watchdog_probe_interval_s", "loop_watchdog_probe_timeout_s", "loop_watchdog_max_strikes",
"filter_silence_narration",
),
("unauthorized_dm_behavior", "unauthorized_dm_behavior", "presence", None, _dm_behavior_choice),
)
@@ -153,21 +154,10 @@ def merge_platform_sections(yaml_cfg: dict, gateway_cfg: Any, gw_data: dict) ->
merged["extra"] = merged_extra
platforms_data[plat_name] = merged
gateway_platforms = gateway_cfg.get("platforms") if isinstance(gateway_cfg, dict) else None
merge(gateway_platforms)
nested_gateway = gateway_cfg if isinstance(gateway_cfg, dict) else {}
merge(nested_gateway.get("platforms"))
merge(yaml_cfg.get("platforms"))
if isinstance(gateway_cfg, dict):
nested: dict = {}
for key, value in gateway_cfg.items():
if key == "platforms":
continue
try:
Platform(key)
except (ValueError, AttributeError):
continue
if isinstance(value, dict):
nested[key] = value
merge(nested)
merge({k: v for k, v in nested_gateway.items() if k != "platforms" and isinstance(v, dict) and _is_platform_name(k)})
api_plat = platforms_data.get("api_server")
if isinstance(api_plat, dict):
@@ -178,6 +168,14 @@ def merge_platform_sections(yaml_cfg: dict, gateway_cfg: Any, gw_data: dict) ->
return platforms_data
def _is_platform_name(key: Any) -> bool:
try:
Platform(key)
except (ValueError, AttributeError):
return False
return True
def platform_section(yaml_cfg: dict, name: str, gateway_platforms: Any) -> tuple:
"""Return ``(section, is_toplevel)`` for platform *name*.
@@ -199,43 +197,32 @@ def _str_keyed(value: Any) -> Any:
return {str(k): v for k, v in value.items()} if isinstance(value, dict) else value
def _dm_behavior(gw_data: dict):
return lambda v: _normalize_choice(v, {"pair", "ignore"}, gw_data.get("unauthorized_dm_behavior", "pair"))
_TELEGRAM = frozenset({Platform.TELEGRAM})
_DISCORD_SLACK = frozenset({Platform.DISCORD, Platform.SLACK})
# Keys copied from a platform's YAML section into ``extra`` (``None`` = every platform);
# third element transforms the value.
def _plain(*keys: str) -> tuple:
"""Keys copied verbatim into ``extra`` for every platform."""
return tuple((k, None, None) for k in keys)
# (key, platforms-or-None, transform-or-None), in ``extra`` insertion order.
# ``"dm"`` defers to the global unauthorized_dm_behavior.
_SHARED_KEYS: tuple = (
("unauthorized_dm_behavior", None, "dm"),
("notice_delivery", None, lambda v: _normalize_choice(v, {"public", "private"}, "public")),
("reply_prefix", None, None),
("reply_in_thread", None, None),
("cron_continuable_surface", None, None),
("require_mention", None, None),
("send_read_receipts", None, None),
*_plain("reply_prefix", "reply_in_thread", "cron_continuable_surface", "require_mention", "send_read_receipts"),
("allowed_chats", _TELEGRAM, None),
("group_allowed_chats", _TELEGRAM, None),
("allowed_topics", _TELEGRAM, None),
("free_response_channels", None, None),
("mention_patterns", None, None),
("exclusive_bot_mentions", None, None),
*_plain("free_response_channels", "mention_patterns", "exclusive_bot_mentions"),
("observe_unmentioned_group_messages", _TELEGRAM, None),
("dm_policy", None, None),
("allow_from", None, None),
("allow_admin_from", None, None),
("user_allowed_commands", None, None),
("group_policy", None, None),
("group_allow_from", None, None),
("group_allow_admin_from", None, None),
("group_user_allowed_commands", None, None),
*_plain(
"dm_policy", "allow_from", "allow_admin_from", "user_allowed_commands",
"group_policy", "group_allow_from", "group_allow_admin_from", "group_user_allowed_commands",
),
("channel_skill_bindings", _DISCORD_SLACK, None),
("channel_prompts", None, _str_keyed),
("gateway_restart_notification", None, None),
("typing_indicator", None, None),
("typing_status_text", None, None),
*_plain("gateway_restart_notification", "typing_indicator", "typing_status_text"),
)
# Top-level port/host/secret bridged into ``extra`` for adapters that read them from config.extra.
@@ -254,8 +241,9 @@ def _bridged_keys(plat: Platform, platform_cfg: dict, gw_data: dict) -> dict:
if key not in platform_cfg or (only is not None and plat not in only):
continue
if transform == "dm":
transform = _dm_behavior(gw_data)
bridged[key] = transform(platform_cfg[key]) if transform else platform_cfg[key]
bridged[key] = _dm_behavior_choice(platform_cfg[key], gw_data.get("unauthorized_dm_behavior", "pair"))
else:
bridged[key] = transform(platform_cfg[key]) if transform else platform_cfg[key]
for key in _PORT_BRIDGE_KEYS.get(plat, ()):
if key in platform_cfg and key not in platform_cfg.get("extra", {}):
bridged[key] = platform_cfg[key]
@@ -285,7 +273,7 @@ def bridge_platform_shared_keys(
``merge_platform_sections`` already merged it with the correct precedence, and re-applying it here
would overwrite that. An explicit top-level enable/disable sets ``_enabled_explicit`` so the env
pass honors ``enabled: false`` for migrated plugin platforms instead of re-enabling them on
token/SDK presence (#41112).
token/SDK presence.
"""
for plat in targets:
if plat == Platform.LOCAL:
@@ -315,7 +303,7 @@ def bridge_platform_shared_keys(
def apply_plugin_yaml_hooks(yaml_cfg: dict, gateway_platforms: Any, platforms_data: dict, registry) -> None:
"""Plugin-owned YAML→env config bridges (``PlatformEntry.apply_yaml_config_fn``, #24836).
"""Plugin-owned YAML→env config bridges (``PlatformEntry.apply_yaml_config_fn``).
Order: shared-key loop → this dispatch → core-only bridges (require_mention/signal) →
``_apply_env_overrides()`` after ``GatewayConfig.from_dict``.
@@ -333,20 +321,16 @@ def apply_plugin_yaml_hooks(yaml_cfg: dict, gateway_platforms: Any, platforms_da
except Exception as e:
logger.debug("apply_yaml_config_fn for %s raised: %s", entry.name, e)
continue
if not isinstance(seeded, dict) or not seeded:
continue
_dict_slot(_dict_slot(platforms_data, entry.name), "extra").update(seeded)
if isinstance(seeded, dict) and seeded:
_dict_slot(_dict_slot(platforms_data, entry.name), "extra").update(seeded)
def bridge_core_env_settings(yaml_cfg: dict, platforms_data: dict) -> None:
"""The two YAML→env bridges that stay in core.
Slack/Telegram/WhatsApp/DingTalk/Mattermost/Matrix/Feishu settings→env bridges migrated to
each plugin's ``apply_yaml_config_fn`` hook (#41112 / #3823 / #25443).
"""The two YAML→env bridges that stay in core (the per-platform ones live in plugin hooks).
Top-level ``require_mention`` → Telegram when the ``telegram:`` section has none: users write it
alongside ``group_sessions_per_user`` expecting it to work (#3979). It keys off the TOP-LEVEL key,
so the telegram plugin's hook (which only runs when a telegram block exists) can't cover it.
alongside ``group_sessions_per_user`` expecting it to work. It keys off the TOP-LEVEL key, so
the telegram plugin's hook (which only runs when a telegram block exists) can't cover it.
Signal ``require_mention`` → ``SIGNAL_REQUIRE_MENTION`` (env wins when already set).
"""
tl_require_mention = yaml_cfg.get("require_mention")
gateway/control_socket.py
+74 -195
View File
@@ -1,18 +1,10 @@
"""Gateway control socket — the gateway-owned local coordination surface.
Instead of process-table scans or ``gateway_state.json`` (which can outlive
its writer), the gateway serves a local-only socket answering versioned JSON
verbs (``identify``, ``status``). A connectable socket with a well-formed
``identify`` answer IS liveness — no PID-reuse heuristics.
Transport — never a TCP port; filesystem/pipe ACLs are the auth boundary.
POSIX: ``$HERMES_HOME/gateway.sock``, or a temp-dir socket plus a
``gateway.sock.path`` pointer file when the home path exceeds ``sun_path``.
Windows: named pipe ``\\\\.\\pipe\\hermes-gateway-<home-hash>``.
Wire contract: ONE request per connection — one JSON line in, one out, then
the server closes (no keep-alive/pipelining). Consumers PREFER the socket and
fall back to the state-file/scan layer when it doesn't answer.
"""Gateway control socket — the gateway-owned local coordination surface: a local-only socket answering
versioned JSON verbs (``identify``, ``status``). A connectable socket with a well-formed ``identify``
answer IS liveness — no PID-reuse heuristics. Never a TCP port: filesystem/pipe ACLs are the auth
boundary. POSIX: ``$HERMES_HOME/gateway.sock`` (or a temp-dir socket + ``gateway.sock.path`` pointer
file when the home path exceeds ``sun_path``); Windows: named pipe ``\\\\.\\pipe\\hermes-gateway-<hash>``.
Wire contract: ONE request per connection — one JSON line in, one out, then the server closes.
Consumers PREFER the socket and fall back to the state-file/scan layer when it doesn't answer.
"""
from __future__ import annotations
@@ -33,28 +25,18 @@ from typing import Any, Callable, Optional
logger = logging.getLogger(__name__)
CONTROL_PROTOCOL_VERSION = 1
_SOCKET_FILENAME = "gateway.sock"
_POINTER_FILENAME = "gateway.sock.path"
_IS_WINDOWS = sys.platform == "win32"
# sun_path limit is 104 on macOS/BSD, 108 on Linux; leave margin for the NUL.
_MAX_UNIX_PATH = 100
_MAX_UNIX_PATH = 100 # sun_path limit is 104 on macOS/BSD, 108 on Linux; margin for the NUL
# Single-line JSON in/out; bounded so a misbehaving peer can't balloon memory.
_MAX_REQUEST_BYTES = 64 * 1024
_MAX_RESPONSE_BYTES = 512 * 1024
_DEFAULT_CLIENT_TIMEOUT = 2.0
# ---------------------------------------------------------------------------
# Path resolution (shared by server and client)
# ---------------------------------------------------------------------------
def _home_hash(home: Path) -> str:
canonical = os.path.normcase(str(Path(home).expanduser().resolve(strict=False)))
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()[:16]
return hashlib.sha256(os.path.normcase(str(Path(home).expanduser().resolve(strict=False))).encode("utf-8")).hexdigest()[:16]
def windows_pipe_name(home: Path) -> str:
@@ -67,29 +49,18 @@ def _fits_sun_path(path: Path) -> bool:
def _fallback_socket_path(home: Path) -> Path:
"""Short temp-dir path for homes whose direct socket path exceeds sun_path.
Tries ``tempfile.gettempdir()`` then ``/tmp`` (POSIX). If nothing fits the
tempdir candidate is returned anyway — bind fails non-fatally and
consumers use the scan layer.
"""
"""Short temp-dir path for homes whose direct socket path exceeds sun_path: ``tempfile.gettempdir()``
then ``/tmp`` (POSIX); if nothing fits the tempdir candidate is returned anyway — bind fails
non-fatally and consumers use the scan layer."""
name = f"hermes-gw-{_home_hash(home)}.sock"
candidates = [Path(tempfile.gettempdir()) / name]
if not _IS_WINDOWS:
candidates.append(Path("/tmp") / name)
for candidate in candidates:
if _fits_sun_path(candidate):
return candidate
return candidates[0]
candidates = [Path(tempfile.gettempdir()) / name] + ([] if _IS_WINDOWS else [Path("/tmp") / name])
return next((c for c in candidates if _fits_sun_path(c)), candidates[0])
def resolve_server_socket_path(home: Path) -> tuple[Path, Optional[Path]]:
"""Return ``(bind_path, pointer_file)``; pointer_file is non-None only
when the temp-dir fallback is in use and the server must record it."""
"""Return ``(bind_path, pointer_file)``; pointer_file is set only for the temp-dir fallback."""
direct = Path(home) / _SOCKET_FILENAME
if _fits_sun_path(direct):
return direct, None
return _fallback_socket_path(home), Path(home) / _POINTER_FILENAME
return (direct, None) if _fits_sun_path(direct) else (_fallback_socket_path(home), Path(home) / _POINTER_FILENAME)
def resolve_client_socket_path(home: Path) -> Optional[Path]:
@@ -97,98 +68,60 @@ def resolve_client_socket_path(home: Path) -> Optional[Path]:
direct = Path(home) / _SOCKET_FILENAME
if direct.exists():
return direct
pointer = Path(home) / _POINTER_FILENAME
try:
if pointer.is_file():
target = pointer.read_text(encoding="utf-8").strip()
if target and Path(target).exists():
return Path(target)
except OSError:
pass
with contextlib.suppress(OSError):
pointer = Path(home) / _POINTER_FILENAME
target = pointer.read_text(encoding="utf-8").strip() if pointer.is_file() else ""
if target and Path(target).exists():
return Path(target)
return None
# ---------------------------------------------------------------------------
# Default payload builders (import-light; overridable at wiring time)
# ---------------------------------------------------------------------------
def _detect_supervisor() -> str:
"""Supervisor kind for THIS process, declared from its own launch env
(not inferred outside-in like ``_detect_supervisor_for_pid``)."""
"""Supervisor kind for THIS process from its own launch env (not inferred outside-in)."""
env = os.environ
if env.get("INVOCATION_ID"):
return "systemd"
if sys.platform == "darwin" and (
env.get("XPC_SERVICE_NAME", "").startswith("ai.hermes")
or env.get("LAUNCHD_SOCKET")
):
if sys.platform == "darwin" and (env.get("XPC_SERVICE_NAME", "").startswith("ai.hermes")
or env.get("LAUNCHD_SOCKET")):
return "launchd"
if env.get("HERMES_DESKTOP_MANAGED"):
return "desktop"
if "--external-supervisor" in sys.argv:
return "external"
return "manual"
return "external" if "--external-supervisor" in sys.argv else "manual"
def build_identify_payload() -> dict[str, Any]:
"""Default ``identify`` answer, built from gateway.status primitives."""
from gateway.status import (
_build_pid_record,
_get_code_identity_fields,
_profile_label_for_home,
read_runtime_status,
)
from gateway.status import _build_pid_record, _get_code_identity_fields, _profile_label_for_home, read_runtime_status
record = _build_pid_record()
payload: dict[str, Any] = {
"protocol": CONTROL_PROTOCOL_VERSION,
**{k: record.get(k) for k in ("kind", "pid", "start_time", "hermes_home")},
"profile": _profile_label_for_home(record.get("hermes_home") or ""),
"supervisor": _detect_supervisor(),
}
payload.update(_get_code_identity_fields())
# served_profiles (multiplex mode) is stamped into runtime status by the runner.
try:
"supervisor": _detect_supervisor(), **_get_code_identity_fields()}
with contextlib.suppress(Exception):
# served_profiles (multiplex mode) is stamped into runtime status by the runner.
served = (read_runtime_status() or {}).get("served_profiles")
if isinstance(served, list) and served:
payload["served_profiles"] = served
except Exception:
pass
return payload
def build_status_payload() -> dict[str, Any]:
"""Default ``status`` answer — current runtime status, answered live."""
from gateway.status import read_runtime_status
return {**(read_runtime_status() or {}), "protocol": CONTROL_PROTOCOL_VERSION,
"answered_at": time.time(), "answering_pid": os.getpid()}
payload = dict(read_runtime_status() or {})
payload["protocol"] = CONTROL_PROTOCOL_VERSION
payload["answered_at"] = time.time()
payload["answering_pid"] = os.getpid()
return payload
# ---------------------------------------------------------------------------
# Server
# ---------------------------------------------------------------------------
class GatewayControlServer:
"""Gateway-owned control socket server (identify/status, v1).
"""Gateway-owned control socket server (identify/status, v1): ``start()`` after the PID-file claim,
``stop()`` on shutdown. All failures are non-fatal — the gateway never refuses to serve messaging
because its control socket couldn't bind; consumers fall back to the scan layer."""
``start()`` after the PID-file claim, ``stop()`` on shutdown. All failures
are non-fatal — the gateway never refuses to serve messaging because its
control socket couldn't bind; consumers fall back to the scan layer.
"""
def __init__(
self,
home: Optional[Path] = None,
*,
verb_handlers: Optional[dict[str, Callable[[], dict[str, Any]]]] = None,
) -> None:
def __init__(self, home: Optional[Path] = None, *,
verb_handlers: Optional[dict[str, Callable[[], dict[str, Any]]]] = None) -> None:
if home is None:
from gateway.status import _get_process_hermes_home
home = _get_process_hermes_home()
self._home = Path(home)
self._server: Optional[asyncio.AbstractServer] = None
@@ -196,33 +129,24 @@ class GatewayControlServer:
self._bind_path: Optional[Path] = None
self._pointer_file: Optional[Path] = None
self._handlers: dict[str, Callable[[], dict[str, Any]]] = {
"identify": build_identify_payload,
"status": build_status_payload,
}
if verb_handlers:
self._handlers.update(verb_handlers)
# -- lifecycle ---------------------------------------------------------
"identify": build_identify_payload, "status": build_status_payload, **(verb_handlers or {})}
async def start(self) -> bool:
"""Bind and start serving. Returns True on success, False otherwise."""
try:
if _IS_WINDOWS:
return await self._start_windows()
return await self._start_posix()
return await (self._start_windows() if _IS_WINDOWS else self._start_posix())
except Exception as exc:
logger.warning("Gateway control socket failed to start (non-fatal): %s", exc)
return False
async def _start_posix(self) -> bool:
bind_path, pointer_file = resolve_server_socket_path(self._home)
# We only get here after winning the PID-file O_EXCL race, so any
# existing file is stale or a collision — never a live sibling.
# We only get here after winning the PID-file O_EXCL race, so any existing
# file is stale or a collision — never a live sibling.
with contextlib.suppress(OSError):
if bind_path.exists():
bind_path.unlink()
# Restrictive umask so the socket is never world-connectable, even
# for the instant before chmod.
# Restrictive umask so the socket is never world-connectable, even for the instant before chmod.
old_umask = os.umask(0o177)
try:
self._server = await asyncio.start_unix_server(self._handle_connection, path=str(bind_path))
@@ -241,10 +165,8 @@ class GatewayControlServer:
loop = asyncio.get_running_loop()
start_serving_pipe = getattr(loop, "start_serving_pipe", None)
if start_serving_pipe is None:
logger.debug(
"Event loop %s has no start_serving_pipe — control socket "
"disabled (selector loop on Windows).", type(loop).__name__,
)
logger.debug("Event loop %s has no start_serving_pipe — control socket "
"disabled (selector loop on Windows).", type(loop).__name__)
return False
pipe_name = windows_pipe_name(self._home)
servers = await start_serving_pipe(lambda: _PipeControlProtocol(self), pipe_name)
@@ -258,48 +180,34 @@ class GatewayControlServer:
self._server.close()
with contextlib.suppress(Exception):
await self._server.wait_closed()
self._server = None
if self._pipe_server is not None:
with contextlib.suppress(Exception):
self._pipe_server.close()
self._pipe_server = None
self._server = self._pipe_server = None
self.cleanup_files()
def cleanup_files(self) -> None:
"""Best-effort removal of socket + pointer files (atexit-safe)."""
for path in (self._bind_path, self._pointer_file):
if path is not None:
with contextlib.suppress(OSError):
path.unlink(missing_ok=True)
# -- request handling ----------------------------------------------------
for path in filter(None, (self._bind_path, self._pointer_file)):
with contextlib.suppress(OSError):
path.unlink(missing_ok=True)
def handle_request_line(self, raw: bytes) -> bytes:
"""One JSON request line -> one JSON response line. Never raises.
Shared by the POSIX stream handler and the Windows pipe protocol.
"""
"""One JSON request line -> one JSON response line. Never raises (shared by POSIX + pipe)."""
request_id: Any = None
try:
request = json.loads(raw.decode("utf-8"))
if not isinstance(request, dict):
raise ValueError("request must be a JSON object")
request_id = request.get("id")
verb = request.get("verb")
request_id, verb = request.get("id"), request.get("verb")
handler = self._handlers.get(verb) if isinstance(verb, str) else None
if handler is None:
response: dict[str, Any] = {
"ok": False, "error": f"unknown verb: {verb!r}",
"protocol": CONTROL_PROTOCOL_VERSION,
"supported_verbs": sorted(self._handlers),
}
response: dict[str, Any] = {"ok": False, "error": f"unknown verb: {verb!r}",
"protocol": CONTROL_PROTOCOL_VERSION, "supported_verbs": sorted(self._handlers)}
else:
response = {"ok": True, "protocol": CONTROL_PROTOCOL_VERSION, "result": handler()}
except Exception as exc:
response = {
"ok": False, "error": f"{type(exc).__name__}: {exc}",
"protocol": CONTROL_PROTOCOL_VERSION,
}
response = {"ok": False, "error": f"{type(exc).__name__}: {exc}", "protocol": CONTROL_PROTOCOL_VERSION}
if request_id is not None:
response["id"] = request_id
try:
@@ -315,10 +223,10 @@ class GatewayControlServer:
raw = await asyncio.wait_for(reader.readline(), timeout=_DEFAULT_CLIENT_TIMEOUT)
if not raw or len(raw) > _MAX_REQUEST_BYTES:
return
# Handlers read disk; keep that off the loop that drives every
# platform adapter so a fast-polling consumer can't stall heartbeats.
loop = asyncio.get_running_loop()
response = await loop.run_in_executor(None, self.handle_request_line, raw.rstrip(b"\n"))
# Handlers read disk; keep that off the loop that drives every platform
# adapter so a fast-polling consumer can't stall heartbeats.
response = await asyncio.get_running_loop().run_in_executor(
None, self.handle_request_line, raw.rstrip(b"\n"))
writer.write(response)
await writer.drain()
except (asyncio.TimeoutError, ConnectionError, OSError):
@@ -332,7 +240,6 @@ class GatewayControlServer:
class _PipeControlProtocol(asyncio.Protocol):
"""One-shot request/response protocol for the Windows named pipe."""
def __init__(self, server: GatewayControlServer) -> None:
self._server = server
self._transport: Any = None
@@ -345,43 +252,25 @@ class _PipeControlProtocol(asyncio.Protocol):
self._buffer.extend(data)
if len(self._buffer) > _MAX_REQUEST_BYTES:
self._transport.close()
return
if b"\n" in self._buffer:
line, _, _ = bytes(self._buffer).partition(b"\n")
elif b"\n" in self._buffer:
try:
self._transport.write(self._server.handle_request_line(line))
self._transport.write(self._server.handle_request_line(bytes(self._buffer).partition(b"\n")[0]))
finally:
self._transport.close()
# ---------------------------------------------------------------------------
# Client (synchronous — used by CLI/updater consumers)
# ---------------------------------------------------------------------------
def query_gateway_control(
home: Path, verb: str, *, timeout: float = _DEFAULT_CLIENT_TIMEOUT
) -> Optional[dict[str, Any]]:
"""Ask the gateway serving ``home`` a control verb; None when unanswered.
Returns the verb's ``result`` payload. Any failure (no/stale socket,
timeout, malformed answer, ``ok: false``) returns None so callers fall
back to the scan layer. Never raises.
"""
def query_gateway_control(home: Path, verb: str, *, timeout: float = _DEFAULT_CLIENT_TIMEOUT) -> Optional[dict[str, Any]]:
"""Ask the gateway serving ``home`` a control verb; returns its ``result`` payload. Any failure (no/stale
socket, timeout, malformed answer, ``ok: false``) returns None so callers fall back to the scan layer.
Never raises."""
request = json.dumps({"verb": verb, "id": 1, "protocol": CONTROL_PROTOCOL_VERSION}).encode("utf-8") + b"\n"
query = _query_windows_pipe if _IS_WINDOWS else _query_unix_socket
try:
raw = query(Path(home), request, timeout)
response = json.loads(raw.decode("utf-8")) if raw else None
except Exception:
return None
if not raw:
return None
try:
response = json.loads(raw.decode("utf-8"))
except (ValueError, UnicodeDecodeError):
return None
if not isinstance(response, dict) or response.get("ok") is not True:
return None
result = response.get("result")
result = response.get("result") if isinstance(response, dict) and response.get("ok") is True else None
return result if isinstance(result, dict) else None
@@ -390,32 +279,25 @@ def _read_response_line(read: Callable[[], bytes], deadline: float) -> Optional[
chunks: list[bytes] = []
while time.monotonic() < deadline:
chunk = read()
if not chunk:
break
chunks.append(chunk)
if b"\n" in chunk:
if not chunk or b"\n" in chunk:
break
if sum(len(c) for c in chunks) > _MAX_RESPONSE_BYTES:
return None
line, _, _ = b"".join(chunks).partition(b"\n")
return line or None
return b"".join(chunks).partition(b"\n")[0] or None
def _query_unix_socket(home: Path, request: bytes, timeout: float) -> Optional[bytes]:
path = resolve_client_socket_path(home)
if path is None:
return None
with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as sock:
# OSError covers ConnectionRefusedError / FileNotFoundError on connect and socket.timeout on read.
with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as sock, contextlib.suppress(OSError):
sock.settimeout(timeout)
try:
sock.connect(str(path))
except (ConnectionRefusedError, FileNotFoundError, OSError):
return None
sock.connect(str(path))
sock.sendall(request)
try:
return _read_response_line(lambda: sock.recv(65536), time.monotonic() + timeout)
except socket.timeout:
return None
return _read_response_line(lambda: sock.recv(65536), time.monotonic() + timeout)
return None
def _query_windows_pipe(home: Path, request: bytes, timeout: float) -> Optional[bytes]: # pragma: no cover - wine2e lane
@@ -446,10 +328,7 @@ def identify_gateway(home: Path, *, timeout: float = _DEFAULT_CLIENT_TIMEOUT) ->
def pause_gateway_for_update(home: Path, *, timeout: float = _DEFAULT_CLIENT_TIMEOUT) -> Optional[dict[str, Any]]:
"""Ask the gateway serving ``home`` to drain and exit for an update.
Returns the ACK ``{"pausing", "already_stopping", "pid", "drain_timeout"}``
or None when no gateway answers (old gateway without the verb, no/dead
socket) — the caller then uses the legacy signal/tree-kill pause path.
"""
"""Ask the gateway serving ``home`` to drain and exit for an update. Returns the ACK ``{"pausing",
"already_stopping", "pid", "drain_timeout"}`` or None when no gateway answers (old gateway without
the verb, no/dead socket) — the caller then uses the legacy signal/tree-kill pause path."""
return query_gateway_control(home, "pause-for-update", timeout=timeout)
gateway/cwd_placeholder.py
+3 -7
View File
@@ -22,16 +22,12 @@ def resolve_placeholder_terminal_cwd(
) -> str | None:
"""Return the ``TERMINAL_CWD`` value to set, or ``None`` to leave it unset.
Cases:
- **local** + placeholder → ``MESSAGING_CWD`` or ``home_fallback``
- **docker** + placeholder + mount on + host ``MESSAGING_CWD`` → host path
(for ``terminal_tool`` ``/workspace`` mapping)
- **docker** + placeholder + mount off → ``None`` (sandbox default)
- other non-local backends + placeholder → ``None``
local + placeholder → ``MESSAGING_CWD`` or ``home_fallback``; docker +
placeholder + mount on + host ``MESSAGING_CWD`` → that host path (for the
``/workspace`` mapping); any other non-local backend → ``None`` (sandbox default).
"""
if configured_cwd and configured_cwd not in CWD_PLACEHOLDERS:
return configured_cwd
backend = (terminal_backend or "local").strip().lower()
messaging = (messaging_cwd or "").strip()
if backend == "local":
gateway/dead_targets.py
+40 -61
View File
@@ -1,18 +1,9 @@
"""Persistent registry of delivery targets that are confirmed unreachable.
When a platform reports a target chat is permanently gone (deleted group,
bot kicked/blocked, deactivated user), re-sending on every cron tick wastes a
send against flood control and spams logs. The delivery layer short-circuits
targets proven dead; any later successful send clears the flag (self-healing).
Scope is deliberately narrow: only *whole-chat* deaths (``forbidden`` and
chat-level ``not_found``) are recorded. Thread/topic-level ``not_found`` is
NOT — adapters self-heal that by retrying without ``reply_to``, and a deleted
topic does not mean the parent chat is dead.
Storage is a JSON file under the active profile's HERMES_HOME (each profile
keeps its own dead set). Reads/writes are best-effort: a corrupt or unwritable
file degrades to in-memory-only rather than raising on the delivery path.
"""Persistent registry of delivery targets confirmed unreachable. Re-sending to a permanently gone chat
(deleted group, bot kicked/blocked, deactivated user) every cron tick wastes flood-control budget; delivery
short-circuits targets proven dead and any later successful send clears the flag. Only *whole-chat* deaths
(``forbidden``, chat-level ``not_found``) are recorded: adapters self-heal thread/topic-level ``not_found``
by retrying without ``reply_to``. Storage is a per-profile JSON file; reads/writes are best-effort (a
corrupt or unwritable file degrades to in-memory-only rather than raising on the delivery path).
"""
from __future__ import annotations
@@ -28,8 +19,7 @@ from hermes_cli.config import get_hermes_home
logger = logging.getLogger(__name__)
# Error kinds (gateway.platforms.base.classify_send_error) meaning the whole
# chat is unreachable, not a transient or thread-level problem.
# classify_send_error kinds meaning the whole chat is unreachable (not transient / thread-level).
_DEAD_ERROR_KINDS = frozenset({"forbidden", "not_found"})
@@ -38,29 +28,35 @@ def _normalize(platform: str, chat_id: str) -> str:
return f"{str(platform).strip().lower()}:{str(chat_id).strip()}"
class DeadTargetRegistry:
"""Thread-safe, persistent set of confirmed-dead targets keyed ``platform:chat_id``.
def classify_dead_error(error_text: Optional[str]) -> Optional[str]:
"""Best-effort dead-target error_kind from a raised error's text, else None. ``_deliver_to_platform``
raises on hard failure (no SendResult), so ``deliver()`` only has the exception string. ``not_found``
collapses chat-level and thread/topic/message-level failures: only a whole-chat not_found means the
target is dead — a deleted forum topic or edited-away message must not mark the entire chat dead."""
if not error_text:
return None
try:
from .platforms.base import classify_send_error, is_chat_level_not_found
except Exception: # pragma: no cover - import guard
return None
kind = classify_send_error(None, error_text=error_text)
dead = kind in _DEAD_ERROR_KINDS and (kind != "not_found" or is_chat_level_not_found(error_text=error_text))
return kind if dead else None
Each entry stores reason + timestamp for observability; :meth:`clear`
(called on a successful send) removes the flag.
"""
class DeadTargetRegistry:
"""Thread-safe, persistent set of confirmed-dead targets keyed ``platform:chat_id``. Each entry stores
reason + timestamp for observability; :meth:`clear` (called on a successful send) removes the flag."""
def __init__(self, path: Optional[Path] = None) -> None:
self._lock = threading.RLock()
self._dead: Dict[str, Dict[str, object]] = {}
self._path = path if path is not None else get_hermes_home() / "gateway" / "dead_targets.json"
self._load()
def _load(self) -> None:
try:
if self._path.exists():
raw = json.loads(self._path.read_text(encoding="utf-8"))
if isinstance(raw, dict):
self._dead = {k: v for k, v in raw.items() if isinstance(v, dict)}
raw = json.loads(self._path.read_text(encoding="utf-8")) if self._path.exists() else {}
self._dead = {k: v for k, v in raw.items() if isinstance(v, dict)} if isinstance(raw, dict) else {}
except (OSError, ValueError) as exc:
logger.debug("dead_targets: could not load %s (%s) — starting empty",
self._path, exc)
self._dead = {}
logger.debug("dead_targets: could not load %s (%s) — starting empty", self._path, exc)
def _flush_locked(self) -> None:
try:
@@ -68,42 +64,26 @@ class DeadTargetRegistry:
tmp = self._path.with_suffix(self._path.suffix + ".tmp")
tmp.write_text(json.dumps(self._dead, indent=2), encoding="utf-8")
tmp.replace(self._path)
except OSError as exc:
# Best-effort: keep in-memory state, never break delivery.
except OSError as exc: # best-effort: keep in-memory state, never break delivery
logger.debug("dead_targets: could not persist %s (%s)", self._path, exc)
@staticmethod
def is_dead_error_kind(error_kind: Optional[str]) -> bool:
"""True when ``error_kind`` denotes a permanent whole-chat death."""
return bool(error_kind) and error_kind in _DEAD_ERROR_KINDS
def is_dead(self, platform: str, chat_id: Optional[str]) -> bool:
if not chat_id:
return False
with self._lock:
return _normalize(platform, chat_id) in self._dead
return bool(chat_id) and _normalize(platform, chat_id) in self._dead
def mark_dead(self, platform: str, chat_id: Optional[str],
reason: str = "") -> bool:
def mark_dead(self, platform: str, chat_id: Optional[str], reason: str = "") -> bool:
"""Record a target as confirmed-dead. Returns True if newly added."""
if not chat_id:
return False
key = _normalize(platform, chat_id)
with self._lock:
existed = key in self._dead
self._dead[key] = {
"platform": str(platform).strip().lower(),
"chat_id": str(chat_id),
"reason": str(reason)[:200],
"marked_at": time.time(),
}
self._dead[key] = {"platform": str(platform).strip().lower(), "chat_id": str(chat_id),
"reason": str(reason)[:200], "marked_at": time.time()}
self._flush_locked()
if not existed:
logger.info(
"dead_targets: marked %s as unreachable (%s) — future deliveries "
"to this target will be skipped until a send succeeds",
key, reason or "no reason given",
)
logger.info("dead_targets: marked %s as unreachable (%s) — future deliveries "
"to this target will be skipped until a send succeeds", key, reason or "no reason given")
return not existed
def clear(self, platform: str, chat_id: Optional[str]) -> bool:
@@ -112,9 +92,8 @@ class DeadTargetRegistry:
return False
key = _normalize(platform, chat_id)
with self._lock:
if key in self._dead:
del self._dead[key]
self._flush_locked()
logger.info("dead_targets: cleared %s (delivery succeeded again)", key)
return True
return False
if self._dead.pop(key, None) is None:
return False
self._flush_locked()
logger.info("dead_targets: cleared %s (delivery succeeded again)", key)
return True
gateway/delivery.py
+135 -341
View File
@@ -1,10 +1,5 @@
"""
Delivery routing for cron job outputs and agent responses.
Routes messages by target: explicit ("telegram:123456789"), platform home
channel ("telegram"), origin (back to where the job was created), or local
(saved to files).
"""
"""Delivery routing for cron job outputs and agent responses, by target: explicit ("telegram:123456789"),
platform home channel ("telegram"), origin (back to where the job was created), or local (files)."""
import logging
import os
@@ -16,43 +11,35 @@ from typing import Dict, List, Optional, Any
from hermes_cli.config import get_hermes_home
from .config import Platform, GatewayConfig, PlatformConfig
from .session import SessionSource
from .dead_targets import DeadTargetRegistry, classify_dead_error
logger = logging.getLogger(__name__)
# Cap before gateway-level truncation of cron output for non-chunking platform
# delivery. Telegram's hard API limit is 4096; the headroom covers the "full
# output saved to …" footer. Adapters that split long messages natively
# (BasePlatformAdapter.splits_long_messages) bypass this entirely.
# Cap before gateway-level truncation of cron output for non-chunking platform delivery. Telegram's hard
# API limit is 4096; the headroom covers the "full output saved to …" footer. Adapters that split long
# messages natively (splits_long_messages) bypass this entirely.
MAX_PLATFORM_OUTPUT = 4000
# Matches strings that are *only* a "silence" narration with optional markdown
# wrappers: *(silent)*, _silent_, `silent`, (silent), silent, 🔇, a bare ".",
# "…", and marker-padded variants. Anchored so substantive messages that merely
# *contain* the word "silent" never match.
# Matches strings that are *only* a "silence" narration with optional markdown wrappers (*(silent)*,
# _silent_, 🔇, a bare ".", "…"). Anchored so messages that merely *contain* "silent" never match.
_SILENCE_NARRATION = re.compile(
r'^[\s*_~`]*\(?\s*(silent|silence|no\s+response|no\s+reply)\s*\.?\)?[\s*_~`]*$'
r'|^[\s*_~`]*[\U0001F507\.\u2026]+[\s*_~`]*$',
re.IGNORECASE,
)
_THREAD_ROUTING_KEYS = ("thread_id", "message_thread_id", "direct_messages_topic_id", "telegram_direct_messages_topic_id")
def _is_silence_narration(content: Optional[str]) -> bool:
"""True when ``content`` is *only* a silence-narration token (length-guarded)."""
if not content:
return False
stripped = content.strip()
if not stripped or len(stripped) > 64:
return False
return bool(_SILENCE_NARRATION.match(stripped))
from .config import Platform, GatewayConfig, PlatformConfig
from .session import SessionSource
from .dead_targets import DeadTargetRegistry
stripped = content.strip() if content else ""
return bool(stripped) and len(stripped) <= 64 and bool(_SILENCE_NARRATION.match(stripped))
@dataclass(frozen=True)
class DeliveryTransport:
"""Resolved live transport for one logical delivery platform."""
adapter: Any
config: Optional[PlatformConfig]
transport_platform: Platform
@@ -61,128 +48,59 @@ class DeliveryTransport:
def is_relay(self) -> bool:
return self.transport_platform == Platform.RELAY
async def send(
self,
logical_platform: Platform,
chat_id: str,
content: str,
metadata: Optional[Dict[str, Any]],
) -> Any:
async def send(self, logical_platform: Platform, chat_id: str, content: str,
metadata: Optional[Dict[str, Any]]) -> Any:
"""Send through this transport while preserving the logical platform."""
if self.is_relay:
return await self.adapter.send_for_platform(
logical_platform, chat_id, content, metadata=metadata,
)
return await self.adapter.send(chat_id, content, metadata=metadata)
return await (self.adapter.send_for_platform(logical_platform, chat_id, content, metadata=metadata)
if self.is_relay else self.adapter.send(chat_id, content, metadata=metadata))
def resolve_delivery_transport(
platform: Platform,
config: GatewayConfig,
adapters: Optional[Dict[Platform, Any]],
) -> Optional[DeliveryTransport]:
"""Resolve a logical platform to its live delivery transport.
A concrete native adapter always wins. Relay is eligible only when its
authenticated transport explicitly advertises that it fronts the logical
platform, so restart-time delivery is independent of per-chat caches without
letting Relay hijack unrelated platform targets.
"""
def resolve_delivery_transport(platform: Platform, config: GatewayConfig,
adapters: Optional[Dict[Platform, Any]]) -> Optional[DeliveryTransport]:
"""Resolve a logical platform to its live delivery transport. A concrete native adapter always wins;
Relay is eligible only when its authenticated transport explicitly advertises that it fronts the
logical platform, so restart-time delivery is independent of per-chat caches without letting Relay
hijack unrelated platform targets."""
live_adapters = adapters or {}
native = live_adapters.get(platform)
native_config = config.platforms.get(platform)
native, native_config = live_adapters.get(platform), config.platforms.get(platform)
# Explicitly supplied live adapters with no config block are honored, but an
# explicitly disabled native adapter never shadows an enabled Relay transport.
if native is not None and (native_config is None or native_config.enabled):
return DeliveryTransport(native, native_config, platform)
relay = live_adapters.get(Platform.RELAY)
relay_config = config.platforms.get(Platform.RELAY)
relay, relay_config = live_adapters.get(Platform.RELAY), config.platforms.get(Platform.RELAY)
fronts_platform = getattr(relay, "fronts_platform", None)
if (
relay is not None
and (relay_config is None or relay_config.enabled)
and callable(fronts_platform)
and fronts_platform(platform)
):
if (relay is not None and (relay_config is None or relay_config.enabled)
and callable(fronts_platform) and fronts_platform(platform)):
return DeliveryTransport(relay, relay_config, Platform.RELAY)
return None
def _as_int(value: Any) -> Optional[int]:
try:
return int(value)
except (TypeError, ValueError):
return None
def looks_like_telegram_private_chat_id(chat_id: Optional[str]) -> bool:
"""True when ``chat_id`` is a positive int — Telegram's private-chat shape.
Groups/channels/supergroups use negative IDs. Single source of truth for the
heuristic, reused by the handoff seed path in ``gateway/run.py`` so
handoff-created DM topics key the same way as inbound DM-topic messages.
"""
if chat_id is None:
"""True when ``chat_id`` is a positive int — Telegram's private-chat shape (groups/channels are negative).
Single source of truth, reused by the handoff seed path in ``gateway/run.py`` so handoff-created DM
topics key the same way as inbound DM-topic messages."""
try:
return int(chat_id) > 0
except (TypeError, ValueError):
return False
parsed = _as_int(chat_id)
return parsed is not None and parsed > 0
def _looks_like_int(value: Optional[str]) -> bool:
return value is not None and _as_int(value) is not None
def _result_field(result: Any, name: str, default: Any = None) -> Any:
"""Read ``name`` from a SendResult object or a plain result dict."""
if isinstance(result, dict):
return result.get(name, default)
return getattr(result, name, default)
def _send_result_failed(result: Any) -> bool:
return _result_field(result, "success", True) is False
try:
return int(value) is not None
except (TypeError, ValueError):
return False
def _send_result_error(result: Any) -> Optional[str]:
error = _result_field(result, "error")
return str(error) if error else None
def _is_thread_not_found_delivery_error(result: Any) -> bool:
error = _send_result_error(result)
return bool(error and "thread not found" in error.lower())
def _classify_dead_from_error_text(error_text: Optional[str]) -> Optional[str]:
"""Best-effort dead-target classification from a raised error's text.
``_deliver_to_platform`` raises on hard failure (no SendResult), so the
``deliver()`` loop only has the exception string; reuse the platform-neutral
classifier to recover the error_kind from it.
"""
if not error_text:
return None
try:
from .platforms.base import classify_send_error, is_chat_level_not_found
except Exception: # pragma: no cover - import guard
return None
kind = classify_send_error(None, error_text=error_text)
if not DeadTargetRegistry.is_dead_error_kind(kind):
return None
# ``not_found`` collapses chat-level and thread/topic/message-level failures.
# Only a whole-chat not_found means the target is dead — a deleted forum
# topic or an edited-away message must not mark the entire chat (and all its
# future deliveries) dead (see gateway.dead_targets).
if kind == "not_found" and not is_chat_level_not_found(error_text=error_text):
return None
return kind
"""Error string of a failed SendResult object / plain result dict ("" if none), or None on success."""
get = result.get if isinstance(result, dict) else (lambda name, default=None: getattr(result, name, default))
return None if get("success", True) is not False else str(get("error") or "")
@dataclass
class DeliveryTarget:
"""A single delivery target: "origin", "local", "telegram" (home channel),
or "telegram:123456[:thread]" (specific chat)."""
"""One target: "origin", "local", "telegram" (home channel) or "telegram:123456[:thread]"."""
platform: Platform
chat_id: Optional[str] = None # None means use home channel
thread_id: Optional[str] = None
@@ -192,38 +110,18 @@ class DeliveryTarget:
@classmethod
def parse(cls, target: str, origin: Optional[SessionSource] = None) -> "DeliveryTarget":
"""Parse "origin" | "local" | "<platform>" | "<platform>:<chat_id>[:<thread_id>]"."""
target_stripped = target.strip()
target_lower = target_stripped.lower()
if target_lower == "origin":
if origin:
return cls(
platform=origin.platform,
chat_id=origin.chat_id,
thread_id=origin.thread_id,
is_origin=True,
)
return cls(platform=Platform.LOCAL, is_origin=True)
if target_lower == "local":
return cls(platform=Platform.LOCAL)
# Platform names are case-insensitive; chat/thread ids keep original case.
# Unknown platforms are treated as local.
if ":" in target_stripped:
parts = target_stripped.split(":", 2)
chat_id = parts[1] if len(parts) > 1 else None
thread_id = parts[2] if len(parts) > 2 else None
try:
platform = Platform(parts[0].lower())
except ValueError:
return cls(platform=Platform.LOCAL)
return cls(platform=platform, chat_id=chat_id, thread_id=thread_id, is_explicit=True)
target = target.strip()
if target.lower() == "origin":
return (cls(platform=origin.platform, chat_id=origin.chat_id, thread_id=origin.thread_id, is_origin=True)
if origin else cls(platform=Platform.LOCAL, is_origin=True))
# Platform names are case-insensitive; chat/thread ids keep case. Unknown platforms -> local.
parts = target.split(":", 2)
try:
return cls(platform=Platform(target_lower))
platform = Platform(parts[0].lower())
except ValueError:
return cls(platform=Platform.LOCAL)
return (cls(platform=platform, chat_id=parts[1], thread_id=parts[2] if len(parts) > 2 else None, is_explicit=True)
if len(parts) > 1 else cls(platform=platform))
def to_string(self) -> str:
"""Convert back to string format."""
@@ -231,23 +129,16 @@ class DeliveryTarget:
return "origin"
if self.platform == Platform.LOCAL:
return "local"
if self.chat_id and self.thread_id:
return f"{self.platform.value}:{self.chat_id}:{self.thread_id}"
if self.chat_id:
return f"{self.platform.value}:{self.chat_id}"
return self.platform.value
parts = [self.platform.value, self.chat_id, self.thread_id if self.chat_id else None]
return ":".join(p for p in parts if p)
async def _ensure_named_dm_topic(adapter: Any, chat_id: str, name: str, *, refresh: bool) -> str:
"""Create (or force-recreate) a named Telegram private DM topic; return its thread id."""
verb = "refresh" if refresh else "create"
ensure_dm_topic = getattr(adapter, "ensure_dm_topic", None)
verb, ensure_dm_topic = "refresh" if refresh else "create", getattr(adapter, "ensure_dm_topic", None)
if ensure_dm_topic is None:
raise RuntimeError(f"Telegram adapter cannot {verb} named private DM topics")
if refresh:
thread_id = await ensure_dm_topic(chat_id, name, force_create=True)
else:
thread_id = await ensure_dm_topic(chat_id, name)
thread_id = await ensure_dm_topic(chat_id, name, **({"force_create": True} if refresh else {}))
if not thread_id:
raise RuntimeError(f"Failed to {verb} Telegram private DM topic '{name}'")
return str(thread_id)
@@ -257,251 +148,154 @@ class DeliveryRouter:
"""Resolves delivery targets and dispatches messages to platform adapters."""
def __init__(self, config: GatewayConfig, adapters: Dict[Platform, Any] = None,
dead_targets: Optional[DeadTargetRegistry] = None):
"""``dead_targets``: shared registry of confirmed-unreachable targets;
a profile-local registry is created when omitted."""
dead_targets: Optional[DeadTargetRegistry] = None): # profile-local registry when omitted
self.config = config
self.adapters = adapters or {}
self.output_dir = get_hermes_home() / "cron" / "output"
self.dead_targets = dead_targets or DeadTargetRegistry()
async def deliver(
self,
content: str,
targets: List[DeliveryTarget],
job_id: Optional[str] = None,
job_name: Optional[str] = None,
metadata: Optional[Dict[str, Any]] = None
) -> Dict[str, Any]:
async def deliver(self, content: str, targets: List[DeliveryTarget], job_id: Optional[str] = None,
job_name: Optional[str] = None, metadata: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
"""Deliver content to all targets; returns per-target results keyed by target string."""
results = {}
for target in targets:
# Skip targets proven permanently unreachable (deleted group, blocked
# bot, deactivated user) — re-sending each tick wastes flood-control
# budget. Self-healing: a later successful send clears the flag.
# LOCAL/origin-without-chat targets are never dead-tracked.
# Skip targets proven permanently unreachable (deleted group, blocked bot, deactivated user) —
# re-sending each tick wastes flood-control budget. Self-healing: a later successful send
# clears the flag. LOCAL/origin-without-chat targets are never dead-tracked.
tracked = target.platform != Platform.LOCAL and target.chat_id
if tracked and self.dead_targets.is_dead(target.platform.value, target.chat_id):
logger.info(
"Skipping delivery to known-dead target %s:%s "
"(send to it again to clear)",
target.platform.value, target.chat_id,
)
results[target.to_string()] = {
"success": False,
"skipped": "dead_target",
"error": "target previously confirmed unreachable",
}
logger.info("Skipping delivery to known-dead target %s:%s (send to it again to clear)",
target.platform.value, target.chat_id)
results[target.to_string()] = {"success": False, "skipped": "dead_target",
"error": "target previously confirmed unreachable"}
continue
try:
if target.platform == Platform.LOCAL:
result = self._deliver_local(content, job_id, job_name, metadata)
else:
result = await self._deliver_to_platform(target, content, metadata)
if target.chat_id and not _send_result_failed(result):
if target.chat_id and _send_result_error(result) is None:
self.dead_targets.clear(target.platform.value, target.chat_id)
results[target.to_string()] = {"success": True, "result": result}
except Exception as e:
# Hard failures raise. Record a whole-chat death so future
# deliveries short-circuit.
if tracked:
dead_kind = _classify_dead_from_error_text(str(e))
if dead_kind:
self.dead_targets.mark_dead(
target.platform.value, target.chat_id,
reason=f"{dead_kind}: {str(e)[:120]}",
)
# Hard failures raise. Record a whole-chat death so future deliveries short-circuit.
dead_kind = classify_dead_error(str(e)) if tracked else None
if dead_kind:
self.dead_targets.mark_dead(target.platform.value, target.chat_id,
reason=f"{dead_kind}: {str(e)[:120]}")
results[target.to_string()] = {"success": False, "error": str(e)}
return results
def _deliver_local(
self,
content: str,
job_id: Optional[str],
job_name: Optional[str],
metadata: Optional[Dict[str, Any]]
) -> Dict[str, Any]:
def _deliver_local(self, content: str, job_id: Optional[str], job_name: Optional[str],
metadata: Optional[Dict[str, Any]]) -> Dict[str, Any]:
"""Save content to local files."""
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
output_path = self.output_dir / (job_id or "misc") / f"{timestamp}.md"
output_path.parent.mkdir(parents=True, exist_ok=True)
lines = [
f"# {job_name}" if job_name else "# Delivery Output",
"",
f"**Timestamp:** {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}",
]
if job_id:
lines.append(f"**Job ID:** {job_id}")
for key, value in (metadata or {}).items():
lines.append(f"**{key}:** {value}")
lines += ["", "---", "", content]
lines = [f"# {job_name}" if job_name else "# Delivery Output", "",
f"**Timestamp:** {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}"]
lines += [f"**Job ID:** {job_id}"] if job_id else []
lines += [f"**{key}:** {value}" for key, value in (metadata or {}).items()] + ["", "---", "", content]
output_path.write_text("\n".join(lines), encoding="utf-8")
return {"path": str(output_path), "timestamp": timestamp}
def _save_full_output(self, content: str, job_id: str) -> Path:
"""Save full cron output to disk and return the file path."""
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
out_dir = get_hermes_home() / "cron" / "output"
out_dir.mkdir(parents=True, exist_ok=True)
path = out_dir / f"{job_id}_{timestamp}.txt"
path = get_hermes_home() / "cron" / "output" / f"{job_id}_{timestamp}.txt"
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(content, encoding="utf-8")
return path
def _filter_silence_narration_enabled(self) -> bool:
"""``HERMES_FILTER_SILENCE_NARRATION`` env overrides the
``gateway.filter_silence_narration`` config flag (default True)."""
"""``HERMES_FILTER_SILENCE_NARRATION`` env overrides the ``gateway.filter_silence_narration`` flag."""
env = os.getenv("HERMES_FILTER_SILENCE_NARRATION")
if env is not None:
return env.strip().lower() in ("1", "true", "yes", "on")
return bool(getattr(self.config, "filter_silence_narration", True))
return (bool(getattr(self.config, "filter_silence_narration", True)) if env is None
else env.strip().lower() in ("1", "true", "yes", "on"))
def _cap_oversized_output(self, adapter: Any, content: str, job_id: str) -> str:
"""Audit-save oversized cron output; truncate it for non-chunking adapters.
Two independent decisions: (1) above MAX_PLATFORM_OUTPUT the full output
is always written to disk as an audit trail, best-effort — a failed save
(full disk, permissions) never blocks delivery; (2) non-chunking adapters
get the content truncated with a footer pointing to the saved file, while
``splits_long_messages`` adapters receive the full payload.
"""
"""Audit-save oversized cron output; truncate it for non-chunking adapters. Above MAX_PLATFORM_OUTPUT
the full output is always written to disk as an audit trail, best-effort — a failed save (full disk,
permissions) never blocks delivery. Non-chunking adapters then get the content truncated with a
footer pointing to the saved file; ``splits_long_messages`` adapters receive the full payload."""
if len(content) <= MAX_PLATFORM_OUTPUT:
return content
saved_path: Optional[Path] = None
try:
saved_path = self._save_full_output(content, job_id)
except OSError as exc:
logger.warning(
"Audit save failed for cron output (%d chars, job=%s): %s — "
"delivery proceeds without audit copy",
len(content), job_id, exc,
)
logger.warning("Audit save failed for cron output (%d chars, job=%s): %s — "
"delivery proceeds without audit copy", len(content), job_id, exc)
if getattr(adapter, "splits_long_messages", False):
if saved_path:
logger.info(
"Cron output preserved for chunking adapter (%d chars) — "
"full output saved to %s",
len(content), saved_path,
)
logger.info("Cron output preserved for chunking adapter (%d chars) — "
"full output saved to %s", len(content), saved_path)
return content
# The footer needs a valid path: if the best-effort save failed, retry
# (a failure now is a real delivery problem and propagates).
if saved_path is None:
saved_path = self._save_full_output(content, job_id)
saved_path = saved_path or self._save_full_output(content, job_id)
footer = f"\n\n... [truncated, full output saved to {saved_path}]"
visible = max(0, MAX_PLATFORM_OUTPUT - len(footer))
logger.info(
"Cron output truncated (%d chars) — full output: %s",
len(content), saved_path,
)
return content[:visible] + footer
logger.info("Cron output truncated (%d chars) — full output: %s", len(content), saved_path)
return content[:max(0, MAX_PLATFORM_OUTPUT - len(footer))] + footer
async def _deliver_to_platform(
self,
target: DeliveryTarget,
content: str,
metadata: Optional[Dict[str, Any]]
) -> Dict[str, Any]:
async def _deliver_to_platform(self, target: DeliveryTarget, content: str,
metadata: Optional[Dict[str, Any]]) -> Dict[str, Any]:
"""Deliver content to a messaging platform."""
transport = resolve_delivery_transport(target.platform, self.config, self.adapters)
if transport is None:
raise ValueError(f"No adapter configured for {target.platform.value}")
adapter = transport.adapter
if not target.chat_id:
raise ValueError(f"No chat ID for {target.platform.value} delivery")
adapter = transport.adapter
content = self._cap_oversized_output(adapter, content, (metadata or {}).get("job_id", "unknown"))
content = self._cap_oversized_output(
adapter, content, (metadata or {}).get("job_id", "unknown")
)
# Substrate-level anti-loop guard: drop hallucinated "silence narration"
# (*(silent)*, 🔇, a bare ".") before it reaches any adapter — in
# bot-to-bot channels these mirror back and forth until a model crashes
# with "no content after all retries"; prompt rules drift across
# providers, so this single chokepoint covers every platform. Local/file
# delivery is never filtered (saved silence has no loop risk).
# Cron output is an ARTIFACT, not model chatter: a legitimately terse job
# ("...", a single 🔇) has no mirror loop, and dropping it while returning
# success is how a cron gets logged as delivered with nothing on the
# wire. Cron sends carry job_id in metadata; everything else is filtered.
# Substrate-level anti-loop guard: drop hallucinated "silence narration" (*(silent)*, 🔇, a bare ".")
# before it reaches any adapter — in bot-to-bot channels these mirror back and forth until a model
# crashes with "no content after all retries"; prompt rules drift across providers, so this single
# chokepoint covers every platform. Local/file delivery is never filtered (saved silence has no loop
# risk). Cron output is an ARTIFACT, not model chatter: a legitimately terse job ("...", a single 🔇)
# has no mirror loop, and dropping it while returning success is how a cron gets logged as delivered
# with nothing on the wire. Cron sends carry job_id in metadata; everything else is filtered.
is_cron_artifact = "job_id" in (metadata or {})
if (
self._filter_silence_narration_enabled()
and not is_cron_artifact
and _is_silence_narration(content)
):
logger.warning(
"Dropped silence-narration outbound to %s (chat=%s): %r",
target.platform.value,
target.chat_id,
content[:40],
)
return {
"success": True,
"filtered": "silence_narration",
"delivered": False,
}
if self._filter_silence_narration_enabled() and not is_cron_artifact and _is_silence_narration(content):
logger.warning("Dropped silence-narration outbound to %s (chat=%s): %r",
target.platform.value, target.chat_id, content[:40])
return {"success": True, "filtered": "silence_narration", "delivered": False}
send_metadata = dict(metadata or {})
if transport.is_relay:
home = self.config.get_home_channel(target.platform)
if home is not None and home.chat_id == target.chat_id:
if home.user_id:
send_metadata["user_id"] = home.user_id
if home.scope_id:
send_metadata["scope_id"] = home.scope_id
home = self.config.get_home_channel(target.platform) if transport.is_relay else None
if home is not None and home.chat_id == target.chat_id:
send_metadata.update({k: v for k, v in (("user_id", home.user_id), ("scope_id", home.scope_id)) if v})
# Caller-supplied thread routing always wins over target.thread_id.
named_telegram_private_topic_name: Optional[str] = None
named_topic: Optional[str] = None # named Telegram private topic created for this send
thread_id = target.thread_id
thread_unrouted = thread_id and not any(
key in send_metadata
for key in ("thread_id", "message_thread_id",
"direct_messages_topic_id", "telegram_direct_messages_topic_id")
)
if thread_unrouted:
is_telegram_private = (
target.platform == Platform.TELEGRAM
and looks_like_telegram_private_chat_id(target.chat_id)
)
if is_telegram_private and not _looks_like_int(thread_id):
# Named topic: create via createForumTopic, use message_thread_id directly.
named_telegram_private_topic_name = thread_id
send_metadata["thread_id"] = await _ensure_named_dm_topic(
adapter, target.chat_id, thread_id, refresh=False
)
send_metadata["telegram_dm_topic_created_for_send"] = True
elif is_telegram_private:
# Legacy numeric private topic ids not created by this send path
# need a reply anchor to stay visible in the requested lane.
if send_metadata.get("telegram_reply_to_message_id") is None:
if thread_id and not any(key in send_metadata for key in _THREAD_ROUTING_KEYS):
send_metadata["thread_id"] = thread_id
if target.platform == Platform.TELEGRAM and looks_like_telegram_private_chat_id(target.chat_id):
if not _looks_like_int(thread_id):
# Named topic: create via createForumTopic, use message_thread_id directly.
named_topic = thread_id
send_metadata["thread_id"] = await _ensure_named_dm_topic(adapter, target.chat_id, thread_id, refresh=False)
send_metadata["telegram_dm_topic_created_for_send"] = True
elif send_metadata.get("telegram_reply_to_message_id") is None:
# Legacy numeric private topic ids not created by this send path need a reply
# anchor to stay visible in the requested lane.
raise RuntimeError(
"Telegram private DM topic delivery requires telegram_reply_to_message_id; "
"send to the bare chat or provide a reply anchor"
)
send_metadata["thread_id"] = thread_id
send_metadata["telegram_dm_topic_reply_fallback"] = True
else:
send_metadata["thread_id"] = thread_id
else:
send_metadata["telegram_dm_topic_reply_fallback"] = True
result = await transport.send(
target.platform, target.chat_id, content, metadata=send_metadata or None,
)
if _send_result_failed(result):
if named_telegram_private_topic_name and _is_thread_not_found_delivery_error(result):
send_metadata["thread_id"] = await _ensure_named_dm_topic(
adapter, target.chat_id, named_telegram_private_topic_name, refresh=True
)
send_metadata["telegram_dm_topic_created_for_send"] = True
result = await transport.send(
target.platform, target.chat_id, content, metadata=send_metadata or None,
)
if _send_result_failed(result):
raise RuntimeError(_send_result_error(result) or f"{target.platform.value} delivery failed")
for retry in (False, True):
result = await transport.send(target.platform, target.chat_id, content, metadata=send_metadata or None)
error = _send_result_error(result)
if retry or error is None or not named_topic or "thread not found" not in error.lower():
break
# The named topic vanished under us: recreate it once and resend.
send_metadata["thread_id"] = await _ensure_named_dm_topic(adapter, target.chat_id, named_topic, refresh=True)
send_metadata["telegram_dm_topic_created_for_send"] = True
if error is not None:
raise RuntimeError(error or f"{target.platform.value} delivery failed")
return result
gateway/delivery_ledger.py
+111 -249
View File
@@ -1,37 +1,13 @@
"""Durable delivery-obligation ledger for gateway final responses.
A final agent response generated but not yet confirmed-delivered is the one
artifact the gateway can lose without a trace: the turn already burned its
tokens, the text exists only in a Python local, and a crash / restart between
finalize and platform ACK drops it silently.
Each outbound final response gets a small durable row in the shared ``state.db``
(same conventions as ``tools.async_delegation`` — WAL, owner pid + process-start
liveness, bounded retention). The gateway writes three checkpoints:
record_obligation() state='pending' before any send attempt
mark_attempting() state='attempting' immediately before the await
mark_delivered() / state='delivered' only on SendResult.success
mark_failed() state='failed' on a definitive rejection
On startup ``sweep_recoverable()`` claims rows whose owner is dead and hands
them back for redelivery. After an adapter reconnects without a restart,
``sweep_failed_for_runtime()`` may claim only the same live process's
allowlisted transient failures. Crash semantics are explicit about ambiguity
(never silently resend an ambiguous send):
- ``pending`` — send never started: redeliver plainly, no dup risk.
- ``attempting`` — crashed mid-await: the platform MAY already have it.
Redelivered WITH a visible recovered-reply marker (honest at-least-once).
- ``failed`` — definitively rejected once; restart is a natural retry
boundary. Also carries the marker.
- ``delivered`` — nothing to do; retention prunes.
Poison rows cannot spin: attempts are capped and stale rows expire, both
transitioning to ``abandoned`` (kept briefly, then pruned).
Everything here is best-effort: ledger failures must never block or delay an
actual send. Callers wrap every call in try/except.
"""Durable delivery-obligation ledger for gateway final responses (rows in the shared ``state.db``;
WAL, owner pid + process-start liveness, bounded retention) so a crash between finalize and
platform ACK cannot lose a response silently. Checkpoints: record_obligation() 'pending' before
any send | mark_attempting() 'attempting' right before the await | mark_delivered() 'delivered'
only on SendResult.success | mark_failed() 'failed' on a definitive rejection. Crash semantics
(never silently resend an ambiguous send): pending = never started, redeliver plainly; attempting
= crashed mid-await, platform MAY have it, redeliver WITH a visible recovered marker; failed =
rejected once, restart is a retry boundary, also marked; delivered = prune. Attempts are capped
and stale rows expire, both -> 'abandoned' (kept briefly, then pruned). Everything is
best-effort: ledger failures must never block a send; callers wrap every call in try/except.
"""
from __future__ import annotations
@@ -42,13 +18,12 @@ import os
import sqlite3
import threading
import time
from contextlib import contextmanager
from contextlib import closing, contextmanager
from typing import Any, Dict, Iterator, List, Optional
from hermes_constants import get_hermes_home
logger = logging.getLogger(__name__)
_DB_LOCK = threading.Lock()
# Redelivery policy knobs (deliberately not config — the ledger is gated by
@@ -58,23 +33,15 @@ STALE_AFTER_SECONDS = 24 * 60 * 60
_RETENTION_SECONDS = 7 * 24 * 60 * 60
_MAX_ROWS = 500
# Visible prefix for redeliveries that might duplicate an already-received
# message (crash mid-send / post-rejection retry). Honest at-least-once.
RECOVERED_MARKER = (
"♻️ Recovered reply — the gateway restarted during delivery, "
"so this may be a duplicate:\n\n"
)
# Visible prefixes for redeliveries that might duplicate an already-received message (crash mid-send /
# post-rejection retry) — honest at-least-once. Runtime recovery uses a distinct marker: no restart
# occurred, but a network rejection's acknowledgement can still have been lost independently.
RECOVERED_MARKER = "♻️ Recovered reply — the gateway restarted during delivery, so this may be a duplicate:\n\n"
RECONNECTED_MARKER = ("♻️ Recovered reply — the messaging platform reconnected after the original "
"delivery failed, so this may be a duplicate:\n\n")
# Runtime recovery uses a distinct marker: no restart occurred, but a network
# rejection's acknowledgement can still have been lost independently.
RECONNECTED_MARKER = (
"♻️ Recovered reply — the messaging platform reconnected after the original "
"delivery failed, so this may be a duplicate:\n\n"
)
# Runtime replay is fail-closed: only errors whose send contract proves they are
# transient reconnect failures. Permanent rejects (blocked bot, bad auth, missing
# chat) must not be retried merely because an adapter reconnected.
# Runtime replay is fail-closed: only errors whose send contract proves they are transient reconnect
# failures. Permanent rejects (blocked bot, bad auth, missing chat) must not be retried on reconnect.
_RUNTIME_RETRYABLE_ERRORS = frozenset({"send_path_degraded"})
@@ -89,15 +56,13 @@ def _connect() -> sqlite3.Connection:
try:
_initialize_schema(conn)
except Exception:
# A PRAGMA/DDL failure after connect() must not leak the connection.
conn.close()
conn.close() # a PRAGMA/DDL failure after connect() must not leak the connection
raise
return conn
def _initialize_schema(conn: sqlite3.Connection) -> None:
from hermes_state import apply_wal_with_fallback
apply_wal_with_fallback(conn, db_label="state.db (delivery_ledger)")
conn.execute(
"""CREATE TABLE IF NOT EXISTS delivery_obligations (
@@ -117,8 +82,7 @@ def _initialize_schema(conn: sqlite3.Connection) -> None:
adapter_profile TEXT
)"""
)
columns = {row[1] for row in conn.execute("PRAGMA table_info(delivery_obligations)")}
if "adapter_profile" not in columns:
if "adapter_profile" not in {row[1] for row in conn.execute("PRAGMA table_info(delivery_obligations)")}:
try:
conn.execute("ALTER TABLE delivery_obligations ADD COLUMN adapter_profile TEXT")
except sqlite3.OperationalError as exc:
@@ -129,29 +93,25 @@ def _initialize_schema(conn: sqlite3.Connection) -> None:
@contextmanager
def _transaction() -> Iterator[sqlite3.Connection]:
"""Open a connection, commit/rollback on exit, and ALWAYS close it.
``sqlite3.Connection`` as a context manager only commits/rolls back; it does
not close. ``with _connect()`` alone would leak a connection (and its WAL/SHM
fds) per call until GC — ``record_obligation`` runs on every outbound final
response, so this ledger would exhaust ``RLIMIT_NOFILE`` on a long-running gateway.
"""
"""Open a connection, commit/rollback on exit, and ALWAYS close it: ``sqlite3.Connection`` as a
context manager only commits/rolls back, so ``with _connect()`` alone leaks a connection (and its
WAL/SHM fds) per call — ``record_obligation`` runs on every final response; exhausts RLIMIT_NOFILE."""
conn = _connect()
with closing(conn), conn:
yield conn
def _start_time(pid: int) -> Optional[int]:
try:
with conn:
yield conn
finally:
conn.close()
from gateway.status import get_process_start_time # lazy: tests monkeypatch gateway.status
return get_process_start_time(pid)
except Exception:
return None
def _owner_stamp() -> tuple[int, Optional[int]]:
pid = os.getpid()
try:
from gateway.status import get_process_start_time
return pid, get_process_start_time(pid)
except Exception:
return pid, None
return pid, _start_time(pid)
def _owner_alive(pid: Any, started_at: Any) -> bool:
@@ -162,18 +122,11 @@ def _owner_alive(pid: Any, started_at: Any) -> bool:
pid = int(pid)
except (TypeError, ValueError):
return False
try:
from gateway.status import get_process_start_time
current_start = get_process_start_time(pid)
except Exception:
current_start = None
current_start = _start_time(pid)
if current_start is None:
# Start time unreadable: alive iff the pid exists. Route through the
# cross-platform probe — ``os.kill(pid, 0)`` on Windows is NOT a no-op
# (bpo-14484: it maps to ``GenerateConsoleCtrlEvent(0, pid)`` and could
# Ctrl+C the gateway's own console group). ``_pid_exists`` keeps the
# EPERM-means-alive semantics (pid exists but is owned by another user).
# Start time unreadable: alive iff the pid exists. Route through the cross-platform probe — on Windows
# ``os.kill(pid, 0)`` is NOT a no-op (bpo-14484: it maps to ``GenerateConsoleCtrlEvent(0, pid)`` and could
# Ctrl+C the gateway's own console group). ``_pid_exists`` keeps EPERM-means-alive (pid owned by another user).
try:
from gateway.status import _pid_exists
except Exception:
@@ -181,47 +134,29 @@ def _owner_alive(pid: Any, started_at: Any) -> bool:
return False # never fall back to a raw sig-0 probe on Windows
try:
os.kill(pid, 0) # windows-footgun: ok — POSIX-only fallback branch
except ProcessLookupError:
return False
except PermissionError:
return True
except OSError:
return False
return True
except OSError as exc: # incl. ProcessLookupError; EPERM means the pid exists
return isinstance(exc, PermissionError)
try:
return bool(_pid_exists(pid))
except Exception:
return False
if started_at is None:
return True
try:
return int(current_start) == int(started_at)
return started_at is None or int(current_start) == int(started_at)
except (TypeError, ValueError):
return True
def compute_obligation_id(session_key: str, message_ref: str, content: str) -> str:
"""Stable id: same turn + same content re-records idempotently, while distinct
threads/topics on one chat never collide (session_key carries platform, chat and
thread; ``message_ref`` is the triggering inbound message id)."""
payload = f"{session_key}|{message_ref}|{content}"
return hashlib.sha256(payload.encode("utf-8", "replace")).hexdigest()[:24]
"""Stable id: same turn + same content re-records idempotently, while distinct threads/topics on one
chat never collide (session_key carries platform/chat/thread; ``message_ref`` = inbound message id)."""
return hashlib.sha256(f"{session_key}|{message_ref}|{content}".encode("utf-8", "replace")).hexdigest()[:24]
def record_obligation(
*,
obligation_id: str,
session_key: str,
platform: str,
chat_id: str,
thread_id: Optional[str],
content: str,
adapter_profile: Optional[str] = None,
) -> None:
def record_obligation(*, obligation_id: str, session_key: str, platform: str, chat_id: str,
thread_id: Optional[str], content: str, adapter_profile: Optional[str] = None) -> None:
"""Record a final response as owed to the platform (state='pending')."""
now = time.time()
stored_profile = str(adapter_profile).strip() if adapter_profile else "default"
pid, started = _owner_stamp()
now, (pid, started) = time.time(), _owner_stamp()
with _DB_LOCK, _transaction() as conn:
conn.execute(
"""INSERT OR REPLACE INTO delivery_obligations
@@ -229,10 +164,8 @@ def record_obligation(
content, state, attempts, created_at, updated_at,
owner_pid, owner_started_at, adapter_profile)
VALUES (?, ?, ?, ?, ?, ?, 'pending', 0, ?, ?, ?, ?, ?)""",
(obligation_id, session_key, platform, str(chat_id),
str(thread_id) if thread_id else None, content, now, now,
pid, started, stored_profile),
)
(obligation_id, session_key, platform, str(chat_id), str(thread_id) if thread_id else None,
content, now, now, pid, started, str(adapter_profile).strip() if adapter_profile else "default"))
_prune()
@@ -251,11 +184,9 @@ def mark_failed(obligation_id: str, error: str = "") -> None:
def release_runtime_claim(obligation_id: str, error: str = "") -> bool:
"""Return an unsent runtime claim to ``failed`` without spending an attempt.
Runtime recovery claims before clearing ``resume_pending`` so two reconnect
paths cannot send the same row. If the session flag cannot be cleared, no
send was attempted and the claim must not consume the redelivery budget.
Fail-closed to the exact current process instance and ``attempting`` state.
"""
Runtime recovery claims before clearing ``resume_pending`` so two reconnect paths cannot send the
same row; if the flag cannot be cleared no send was attempted and the claim must not consume the
redelivery budget. Fail-closed to the exact current process instance and ``attempting`` state."""
pid, started = _owner_stamp()
if started is None:
return False
@@ -267,9 +198,7 @@ def release_runtime_claim(obligation_id: str, error: str = "") -> bool:
updated_at=?, last_error=?
WHERE obligation_id=? AND state='attempting'
AND owner_pid IS ? AND owner_started_at IS ?""",
(time.time(), error[:500] if error else None,
obligation_id, pid, started),
)
(time.time(), error[:500] if error else None, obligation_id, pid, started))
return bool(cursor.rowcount)
@@ -279,39 +208,31 @@ def _update_state(obligation_id: str, state: str, error: str = "") -> None:
"""UPDATE delivery_obligations
SET state=?, updated_at=?, last_error=?
WHERE obligation_id=?""",
(state, time.time(), error[:500] if error else None, obligation_id),
)
(state, time.time(), error[:500] if error else None, obligation_id))
def _exhausted(attempts: int, created_at: float, now: float) -> bool:
return attempts >= MAX_ATTEMPTS or (now - created_at) > STALE_AFTER_SECONDS
def _claimed_row(oid, session_key, platform, chat_id, thread_id, content, attempts, profile, *,
needs_marker: bool, runtime: bool = False) -> Dict[str, Any]:
"""Claimed-row dict handed back for redelivery; ``runtime`` adds the reconnect-marker fields."""
return {"obligation_id": oid, "session_key": session_key, "platform": platform, "chat_id": chat_id,
"thread_id": thread_id, "content": content, "needs_marker": needs_marker,
**({"marker": RECONNECTED_MARKER} if runtime else {}), "profile": profile,
**({"runtime_recovery": True} if runtime else {}), "attempts": attempts + 1}
def sweep_recoverable(
now: Optional[float] = None,
*,
deliverable_platforms: Optional[set] = None,
deliverable_targets: Optional[set] = None,
) -> List[Dict[str, Any]]:
def sweep_recoverable(now: Optional[float] = None, *, deliverable_platforms: Optional[set] = None,
deliverable_targets: Optional[set] = None) -> List[Dict[str, Any]]:
"""Claim undelivered rows owned by dead processes; return them for redelivery.
Claiming atomically re-stamps the owner to THIS process and increments
``attempts`` (the UPDATE is guarded on the previous owner stamp, so a second
gateway racing the same sweep cannot double-claim). Rows over the attempts
cap or stale cutoff transition to 'abandoned' instead of being returned.
``deliverable_platforms`` restricts claiming to platforms the caller can
actually send on this boot: ``attempts`` is the redelivery budget and must
only be spent on a real send, else a platform that failed to connect burns
one attempt per boot and hits the cap having never been sent once. Rows for
absent platforms are left untouched (the stale cutoff still bounds them).
``deliverable_targets`` further scopes multiplexed gateways by exact
``(platform, adapter_profile)`` so one connected bot cannot spend another
disconnected bot's retry budget.
"""
now = now if now is not None else time.time()
pid, started = _owner_stamp()
Claiming atomically re-stamps the owner to THIS process and increments ``attempts`` (the UPDATE is
guarded on the previous owner stamp, so a second gateway racing the same sweep cannot double-claim).
Rows over the attempts cap or stale cutoff become 'abandoned'. ``deliverable_platforms`` restricts
claiming to platforms the caller can send on this boot: ``attempts`` is the redelivery budget and
must only be spent on a real send, else a platform that failed to connect burns one attempt per boot
and hits the cap having never been sent once (the stale cutoff still bounds untouched rows).
``deliverable_targets`` further scopes multiplexed gateways by exact ``(platform, adapter_profile)``
so one connected bot cannot spend another disconnected bot's retry budget."""
now, (pid, started) = now if now is not None else time.time(), _owner_stamp()
claimed: List[Dict[str, Any]] = []
with _DB_LOCK, _transaction() as conn:
rows = conn.execute(
@@ -321,73 +242,45 @@ def sweep_recoverable(
FROM delivery_obligations
WHERE state IN ('pending', 'attempting', 'failed')"""
).fetchall()
for (oid, session_key, platform, chat_id, thread_id, content, state,
attempts, created_at, owner_pid, owner_started_at,
adapter_profile) in rows:
for (oid, session_key, platform, chat_id, thread_id, content, state, attempts, created_at,
owner_pid, owner_started_at, adapter_profile) in rows:
if _owner_alive(owner_pid, owner_started_at):
continue # a live gateway still owns this row
if _exhausted(attempts, created_at, now):
if attempts >= MAX_ATTEMPTS or (now - created_at) > STALE_AFTER_SECONDS: # exhausted -> abandoned
conn.execute(
"""UPDATE delivery_obligations
SET state='abandoned', updated_at=? WHERE obligation_id=?""",
(now, oid),
)
SET state='abandoned', updated_at=? WHERE obligation_id=?""", (now, oid))
continue
if deliverable_platforms is not None and platform not in deliverable_platforms:
if ((deliverable_platforms is not None and platform not in deliverable_platforms)
or (deliverable_targets is not None and (platform, adapter_profile) not in deliverable_targets)):
continue # no adapter this boot — claiming would spend an attempt on a no-op
if deliverable_targets is not None and (platform, adapter_profile) not in deliverable_targets:
continue
cursor = conn.execute(
"""UPDATE delivery_obligations
SET owner_pid=?, owner_started_at=?, attempts=attempts+1,
updated_at=?
WHERE obligation_id=? AND (owner_pid IS ? OR owner_pid=?)""",
(pid, started, now, oid, owner_pid, owner_pid),
)
if cursor.rowcount:
claimed.append({
"obligation_id": oid,
"session_key": session_key,
"platform": platform,
"chat_id": chat_id,
"thread_id": thread_id,
"content": content,
# pending = never started, redeliver plainly; else carry marker.
"needs_marker": state != "pending",
"profile": adapter_profile,
"attempts": attempts + 1,
})
(pid, started, now, oid, owner_pid, owner_pid))
if cursor.rowcount: # pending = never started, redeliver plainly; else carry marker
claimed.append(_claimed_row(oid, session_key, platform, chat_id, thread_id, content, attempts,
adapter_profile, needs_marker=state != "pending"))
return claimed
def sweep_failed_for_runtime(
platform: str,
now: Optional[float] = None,
*,
profile: Optional[str] = None,
) -> List[Dict[str, Any]]:
def sweep_failed_for_runtime(platform: str, now: Optional[float] = None, *,
profile: Optional[str] = None) -> List[Dict[str, Any]]:
"""Claim this process's reconnect-retryable failed rows for one adapter.
``profile`` scopes multiplexed gateways to the bot identity that owned the
failed send; ``None`` means the primary/default adapter. The persisted
adapter owner is independent of the routed session namespace. Unowned rows
and rows owned by another process are left for the startup/dead-owner sweep.
Startup recovery ignores rows owned by a live gateway, so a final response
rejected with ``send_path_degraded`` would stay stranded when only the
adapter reconnects. This sweep closes that gap without weakening ownership:
only rows stamped to this exact process instance, only allowlisted transient
errors, same attempts/staleness bounds, every update guarded by the prior
owner stamp and ``failed`` state. Claimed rows always carry the reconnect
marker because the failed send's acknowledgement is not safe to infer.
"""
now = now if now is not None else time.time()
pid, started = _owner_stamp()
if started is None:
# PID alone cannot distinguish this process from a stale row left after
# PID reuse. Runtime replay is optional, so fail closed; startup
# recovery remains the durable fallback.
return []
``profile`` scopes multiplexed gateways to the bot identity that owned the failed send (``None`` =
primary/default adapter); unowned rows and rows owned by another process are left for the
startup/dead-owner sweep. Startup recovery ignores rows owned by a live gateway, so a response
rejected with ``send_path_degraded`` would stay stranded when only the adapter reconnects; this closes
that gap without weakening ownership: only rows stamped to this exact process instance, only
allowlisted transient errors, same attempts/staleness bounds, every update guarded by the prior owner
stamp and ``failed`` state. Claimed rows always carry the reconnect marker (the failed send's ack is
not safe to infer)."""
now, (pid, started) = now if now is not None else time.time(), _owner_stamp()
if started is None: # PID alone cannot distinguish this process from a stale row left after PID
return [] # reuse; runtime replay is optional, so fail closed (startup recovery remains).
expected_profile = "default" if not profile or profile == "default" else str(profile)
claimed: List[Dict[str, Any]] = []
with _DB_LOCK, _transaction() as conn:
@@ -396,66 +289,41 @@ def sweep_failed_for_runtime(
content, attempts, created_at, owner_pid,
owner_started_at, last_error, adapter_profile
FROM delivery_obligations
WHERE state='failed' AND platform=?""",
(platform,),
).fetchall()
for (oid, session_key, row_platform, chat_id, thread_id, content,
attempts, created_at, owner_pid, owner_started_at, last_error,
adapter_profile) in rows:
if adapter_profile != expected_profile:
continue
WHERE state='failed' AND platform=?""", (platform,)).fetchall()
for (oid, session_key, row_platform, chat_id, thread_id, content, attempts, created_at,
owner_pid, owner_started_at, last_error, adapter_profile) in rows:
# Exact process-start matching prevents PID reuse from stealing work.
if owner_pid != pid or owner_started_at != started:
if (adapter_profile != expected_profile or owner_pid != pid or owner_started_at != started
or str(last_error or "").strip().lower() not in _RUNTIME_RETRYABLE_ERRORS):
continue
if str(last_error or "").strip().lower() not in _RUNTIME_RETRYABLE_ERRORS:
continue
owner_guard = (oid, owner_pid, owner_started_at)
if _exhausted(attempts, created_at, now):
owner_guard = (now, oid, owner_pid, owner_started_at)
if attempts >= MAX_ATTEMPTS or (now - created_at) > STALE_AFTER_SECONDS: # exhausted -> abandoned
conn.execute(
"""UPDATE delivery_obligations
SET state='abandoned', updated_at=?
WHERE obligation_id=? AND state='failed'
AND owner_pid IS ? AND owner_started_at IS ?""",
(now, *owner_guard),
)
AND owner_pid IS ? AND owner_started_at IS ?""", owner_guard)
continue
cursor = conn.execute(
"""UPDATE delivery_obligations
SET state='attempting', attempts=attempts+1, updated_at=?
WHERE obligation_id=? AND state='failed'
AND owner_pid IS ? AND owner_started_at IS ?""",
(now, *owner_guard),
)
AND owner_pid IS ? AND owner_started_at IS ?""", owner_guard)
if cursor.rowcount:
claimed.append({
"obligation_id": oid,
"session_key": session_key,
"platform": row_platform,
"chat_id": chat_id,
"thread_id": thread_id,
"content": content,
"needs_marker": True,
"marker": RECONNECTED_MARKER,
"profile": adapter_profile,
"runtime_recovery": True,
"attempts": attempts + 1,
})
claimed.append(_claimed_row(oid, session_key, row_platform, chat_id, thread_id, content,
attempts, adapter_profile, needs_marker=True, runtime=True))
return claimed
def _prune(now: Optional[float] = None) -> None:
now = now if now is not None else time.time()
cutoff = now - _RETENTION_SECONDS
try:
with _transaction() as conn:
conn.execute(
"""DELETE FROM delivery_obligations
WHERE state IN ('delivered', 'abandoned') AND updated_at < ?""",
(cutoff,),
)
WHERE state IN ('delivered', 'abandoned') AND updated_at < ?""", (now - _RETENTION_SECONDS,))
total = conn.execute("SELECT COUNT(*) FROM delivery_obligations").fetchone()[0]
excess = max(0, total - _MAX_ROWS)
if excess:
if total > _MAX_ROWS:
conn.execute(
"""DELETE FROM delivery_obligations WHERE obligation_id IN (
SELECT obligation_id FROM delivery_obligations
@@ -464,9 +332,7 @@ def _prune(now: Optional[float] = None) -> None:
WHEN 'abandoned' THEN 1
ELSE 2
END, updated_at ASC
LIMIT ?)""",
(excess,),
)
LIMIT ?)""", (total - _MAX_ROWS,))
except Exception:
logger.debug("delivery ledger prune failed", exc_info=True)
@@ -476,12 +342,8 @@ def ledger_enabled(config: Optional[Dict[str, Any]] = None) -> bool:
try:
if config is None:
from hermes_cli.config import load_config
config = load_config()
gw = config.get("gateway") or {}
value = gw.get("delivery_ledger", True)
if isinstance(value, str):
return value.strip().lower() not in {"false", "0", "no", "off"}
return bool(value)
value = (config.get("gateway") or {}).get("delivery_ledger", True)
return value.strip().lower() not in {"false", "0", "no", "off"} if isinstance(value, str) else bool(value)
except Exception:
return True
gateway/disk_status.py
+24 -64
View File
@@ -1,86 +1,49 @@
"""Disk-usage rollup for ``/api/status``.
Companion to :mod:`gateway.memory_status`: a hosted agent can fill its data
volume (SQLite writes failing, sessions and config saves lost) while its
dashboard and the NAS agent card look healthy. ``gateway/readiness.py``
already probes disk, but readiness is a component verdict, not user-facing
telemetry — this module produces the public block the dashboard SPA and the
NAS availability sweep consume.
Disk is sampled live via :func:`shutil.disk_usage` (one ``statvfs`` call), so
there is no staleness dimension and no ``sampled_at``.
Public-safety: ``/api/status`` is unauthenticated (``PUBLIC_API_PATHS``); this
block carries only coarse numbers (MB, one-decimal percent) and an enum — the same
disclosure class as the ``memory`` block.
Best-effort and read-only: an unreadable filesystem degrades to
volume (SQLite writes failing, config saves lost) while its dashboard looks
healthy. Sampled live via one ``statvfs`` call, so there is no ``sampled_at``.
``/api/status`` is unauthenticated: only coarse numbers (MB, one-decimal
percent) and an enum. Best-effort: an unreadable filesystem degrades to
``pressure="unknown"`` rather than raising into the status endpoint.
"""
from __future__ import annotations
import logging
import shutil
from pathlib import Path
from typing import Any, Dict, Optional
logger = logging.getLogger(__name__)
# Percent alone misleads both ways: 90% used on 100 GB leaves 10 GB, while 50%
# on a tiny volume is one download from write failures. So percent triggers are
# gated on absolute headroom also being low, and a hard absolute floor applies
# regardless of size (below it SQLite journaling / config writes are at risk).
_CRITICAL_FREE_MB = 256 # < 256 MB free: critical on any volume
_CRITICAL_PERCENT = 95.0 # >= 95% used AND < 1 GB free: critical
_CRITICAL_HEADROOM_MB = 1024
_ELEVATED_FREE_MB = 512 # < 512 MB free: elevated on any volume
_ELEVATED_PERCENT = 85.0 # >= 85% used AND < 4 GB free: elevated
_ELEVATED_HEADROOM_MB = 4096
from gateway.memory_status import _nonneg_int
# Percent alone misleads both ways: 90% used on 100 GB leaves 10 GB, while 50% on a tiny
# volume is one download from write failures. So percent triggers are gated on absolute
# headroom also being low, and a hard floor applies regardless of size (below it SQLite
# journaling / config writes are at risk). Rows: (level, free floor, used-%, headroom); worst first.
_PRESSURE_TIERS = (
("critical", 256, 95.0, 1024), # < 256 MB free, or >= 95% used AND < 1 GB free
("elevated", 512, 85.0, 4096), # < 512 MB free, or >= 85% used AND < 4 GB free
)
_BYTES_PER_MB = 1024 * 1024
def _coerce_mb(value: Any) -> Optional[int]:
if isinstance(value, bool) or not isinstance(value, int) or value < 0:
return None
return value
def classify_disk_pressure(free_mb: Any, total_mb: Any) -> str:
"""Map free/total MB to ``ok``/``elevated``/``critical``.
``unknown`` when the sample is missing or malformed — the caller must
not treat "we could not read it" as "disk is fine".
"""
free = _coerce_mb(free_mb)
total = _coerce_mb(total_mb)
if free is None or total is None or total <= 0:
"""``ok``/``elevated``/``critical`` from free/total MB; ``unknown`` when the sample
is missing/malformed — "could not read it" must never read as "fine"."""
free, total = _nonneg_int(free_mb), _nonneg_int(total_mb)
if free is None or not total:
return "unknown"
used_percent = (1 - free / total) * 100.0
for level, free_floor, percent_floor, headroom in (
("critical", _CRITICAL_FREE_MB, _CRITICAL_PERCENT, _CRITICAL_HEADROOM_MB),
("elevated", _ELEVATED_FREE_MB, _ELEVATED_PERCENT, _ELEVATED_HEADROOM_MB),
):
for level, free_floor, percent_floor, headroom in _PRESSURE_TIERS:
if free < free_floor or (used_percent >= percent_floor and free < headroom):
return level
return "ok"
def collect_disk_status(home: Optional[Path] = None) -> Dict[str, Any]:
"""Build the ``disk`` block for ``/api/status``.
``home`` scopes the sample to a profile's HERMES_HOME (same contract as the
``memory`` block; on hosted images every profile shares one volume).
Always returns a dict and never raises — an unreadable/unmounted filesystem
yields ``{"pressure": "unknown", ...}``.
"""
status: Dict[str, Any] = {
"pressure": "unknown",
"total_mb": None,
"free_mb": None,
"used_percent": None,
}
"""``disk`` block for ``/api/status`` (same ``home`` contract as ``memory``).
Never raises — an unreadable/unmounted filesystem yields ``pressure="unknown"``."""
status: Dict[str, Any] = {"pressure": "unknown", "total_mb": None, "free_mb": None, "used_percent": None}
try:
if home is None:
from hermes_constants import get_hermes_home
@@ -91,10 +54,7 @@ def collect_disk_status(home: Optional[Path] = None) -> Dict[str, Any]:
return status
if usage.total <= 0:
return status
total_mb = usage.total // _BYTES_PER_MB
free_mb = usage.free // _BYTES_PER_MB
status["total_mb"] = total_mb
status["free_mb"] = free_mb
status["used_percent"] = round((usage.used / usage.total) * 100, 1)
status["pressure"] = classify_disk_pressure(free_mb, total_mb)
total_mb, free_mb = usage.total // _BYTES_PER_MB, usage.free // _BYTES_PER_MB
status.update(total_mb=total_mb, free_mb=free_mb, used_percent=round((usage.used / usage.total) * 100, 1),
pressure=classify_disk_pressure(free_mb, total_mb))
return status
gateway/display_config.py
+25 -57
View File
@@ -1,63 +1,42 @@
"""Per-platform display/verbosity configuration resolver.
"""Per-platform display/verbosity resolver (``resolve_display_setting``).
``resolve_display_setting()`` is the single entry-point. Resolution order
(first non-None wins):
1. ``display.platforms.<platform>.<key>`` — explicit per-platform override
2. ``display.<key>`` — global user setting
3. ``_PLATFORM_DEFAULTS[<platform>][<key>]`` — built-in platform default
4. ``_GLOBAL_DEFAULTS[<key>]`` — built-in global default
Exception: ``display.streaming`` is CLI-only. Gateway streaming follows the
top-level ``streaming`` config unless ``display.platforms.<platform>.streaming``
sets an explicit per-platform override.
Backward compatibility: ``display.tool_progress_overrides`` is still read as a
fallback for ``tool_progress`` when no ``display.platforms`` entry exists (a
config migration moves the old format into ``display.platforms``).
Resolution order, first non-None wins: ``display.platforms.<platform>.<key>`` →
``display.<key>`` → ``_PLATFORM_DEFAULTS[platform][key]`` → ``_GLOBAL_DEFAULTS[key]``.
Exception: ``display.streaming`` is CLI-only; gateway streaming follows the top-level
``streaming`` config unless a per-platform override sets it. Legacy
``display.tool_progress_overrides`` is still read as a ``tool_progress`` fallback.
"""
from __future__ import annotations
from typing import Any
# Settings configurable per-platform, with global defaults. Other display
# settings (compact, personality, skin, ...) are CLI-only.
# Settings configurable per-platform; other display settings are CLI-only.
_GLOBAL_DEFAULTS: dict[str, Any] = {
"tool_progress": "all",
"tool_progress_grouping": "accumulate", # "accumulate" = edit one bubble; "separate" = one msg per tool
"show_reasoning": False,
# Reasoning summary rendering: "code" (💭 **Reasoning:** + fenced block),
# "blockquote" ("> " lines), "subtext" ("-# " Discord small grey text).
# "code" (💭 **Reasoning:** + fenced block), "blockquote" ("> "), "subtext" ("-# " Discord).
"reasoning_style": "code",
"tool_preview_length": 0,
"streaming": None, # None = follow top-level streaming config
# Gateway-only assistant/status chatter; default on for back-compat, mobile
# platforms opt down to final-answer-first.
# Gateway-only assistant/status chatter; mobile platforms opt down to final-answer-first.
"interim_assistant_messages": True,
"long_running_notifications": True,
"busy_ack_detail": True,
# busy_input_mode=steer confirmation echo ("Steered into current run").
# Disabling only suppresses the echo; the text still lands in the run.
# busy_input_mode=steer echo ("Steered into current run"); the text still lands in the run.
"busy_steer_ack_enabled": True,
# Delete tool-progress / "⏳ Working" / status bubbles after the final
# response on platforms that support deletion (e.g. Telegram). Off by
# default; progress is still shown live, only cleaned up after success so
# the chat doesn't fill with stale breadcrumbs. Failed runs leave bubbles
# in place as breadcrumbs.
# Delete tool-progress / "⏳ Working" bubbles after a SUCCESSFUL final response on
# platforms that support deletion (Telegram); failed runs keep them as breadcrumbs.
"cleanup_progress": False,
# Live working-state text on platforms whose typing indicator renders text
# (Slack assistant status): "full"/true = verb + argument preview,
# "verb" = verb only (keeps paths/commands out of shared channels),
# "off"/false = static text. Independent of tool_progress and free: the
# existing typing refresh cadence just renders different text.
# Working-state text on text-rendering indicators (Slack assistant status):
# "full"/true = verb + argument preview, "verb" = verb only (keeps paths out of
# shared channels), "off"/false = static text.
"live_status": "full",
}
# Per-platform defaults tiered by capability:
# HIGH — message editing, personal/team use
# MEDIUM — editing, but often workspace/customer-facing
# LOW — no edit support: each progress message is permanent
# MINIMAL — batch / non-interactive delivery
# Tiers: HIGH = editing, personal/team use; MEDIUM = editing but customer-facing;
# LOW = no edit support (progress messages are permanent); MINIMAL = batch delivery.
_TIER_HIGH = {
"tool_progress": "all",
"show_reasoning": False,
@@ -79,8 +58,7 @@ _TIER_LOW = {
_TIER_MINIMAL = {**_TIER_LOW, "tool_preview_length": 0}
_PLATFORM_DEFAULTS: dict[str, dict[str, Any]] = {
# Telegram is usually a mobile inbox: quiet tool_progress and no busy-ack
# iteration counter, but DO surface interim assistant commentary and
# Mobile inbox: quiet tool_progress / busy-ack, but keep interim commentary and
# heartbeats so it doesn't look like "typing..." for 30 minutes.
"telegram": {**_TIER_HIGH, "tool_progress": "off", "busy_ack_detail": False},
# Discord's "-# " subtext reads as metadata, so reasoning defaults to it.
@@ -96,25 +74,18 @@ _PLATFORM_DEFAULTS: dict[str, dict[str, Any]] = {
"mattermost": _TIER_MEDIUM,
"matrix": _TIER_MEDIUM,
"feishu": _TIER_MEDIUM,
# Buzz (Nostr via buzz-cli) can edit in place but channels are shared
# community spaces; without an entry it inherited the verbose globals.
# Buzz (Nostr) can edit in place but channels are shared community spaces.
"buzz": _TIER_MEDIUM,
"signal": _TIER_LOW,
"whatsapp": _TIER_MEDIUM, # Baileys bridge supports /edit
# Cloud API supports editing but our adapter lacks edit_message; promote
# to MEDIUM once it lands.
"whatsapp_cloud": _TIER_LOW,
# Photon and BlueBubbles are permanent-message iMessage inboxes (no edit);
# without an entry Photon inherited the noisy "all" globals and narrated
# on nearly every turn.
"whatsapp_cloud": _TIER_LOW, # adapter lacks edit_message; promote once it lands
# Permanent-message iMessage inboxes (no edit).
"photon": _TIER_LOW,
"bluebubbles": _TIER_LOW,
"weixin": _TIER_LOW,
# WeCom is non-editable but has a native streaming transport (msgtype
# "stream") the consumer routes mid-stream content through; streaming on
# gives the client a typing animation + cumulative updates instead of a
# single one-shot markdown drop.
# Non-editable, but its native "stream" msgtype gives a typing animation +
# cumulative updates instead of a one-shot markdown drop.
"wecom": {**_TIER_LOW, "streaming": True},
"wecom_callback": _TIER_LOW,
"dingtalk": _TIER_LOW,
@@ -155,8 +126,7 @@ def resolve_display_setting(
if isinstance(legacy, dict) and legacy.get(platform_key) is not None:
return _normalise(setting, legacy[platform_key])
# 2. Global user setting. display.streaming controls only CLI terminal
# streaming; gateway streaming follows the top-level config + platform overrides.
# 2. Global user setting (display.streaming is CLI-only, see module docstring).
if setting != "streaming" and display_cfg.get(setting) is not None:
return _normalise(setting, display_cfg[setting])
@@ -167,9 +137,7 @@ def resolve_display_setting(
return fallback if val is None else val
# ---------------------------------------------------------------------------
# Normalisation of YAML quirks (bare ``off`` → False in YAML 1.1, etc.)
# ---------------------------------------------------------------------------
# --- Normalisation of YAML quirks (bare ``off`` → False in YAML 1.1, etc.) ---
_TRUTHY = {"true", "1", "yes", "on"}
_FALSY = {"false", "0", "no"}
gateway/drain_control.py
+54 -124
View File
@@ -1,38 +1,18 @@
"""External drain-control marker contract (dashboard → gateway).
There is no control channel into a running gateway, so the dashboard's
begin/cancel-drain endpoint talks to it the same way ``.restart_notify.json``
does: it writes (or removes) a marker file and a gateway watcher reacts. This
module owns the contract so writer and reader can never disagree.
Contract (presence-based):
* begin-drain → write ``{HERMES_HOME}/.drain_request.json`` with
``{"action": "drain", "requested_at": <iso>, "principal": <str>,
"epoch": <instantiation-epoch>, "suppress_notification": <bool>}``.
* cancel-drain → remove the marker.
* The watcher treats presence of an ACTIVE marker as "external drain": flip
``gateway_state -> "draining"`` and stop accepting new turns. Absence or a
stale marker means "not draining" (revert to ``running``).
Staleness — two independent, individually-lenient signals (either suffices):
* epoch mismatch: HERMES_HOME is a durable volume on Hermes Cloud, so a
marker survives the machine restart that the drain-gated action (update,
migrate, env edit) ends in; honouring it would park the fresh gateway in
``draining`` forever. A marker is stamped with this instantiation's epoch
and ignored only on a *definite* mismatch.
* expiry: a same-epoch orphan (action completed without a restart, writer
never cancelled) is ignored once ``requested_at`` is older than
:data:`DRAIN_REQUEST_MAX_AGE_SECONDS`. Re-calling :func:`write_drain_request`
refreshes the timestamp — the sanctioned keep-alive for a long drain.
Reading never raises: a malformed/half-written file reads as "present but
contentless" (``{}``), which still counts as drain-active — fail-safe toward
quiescing. Both staleness checks only reject on a *definite* verdict; a marker
with no epoch/timestamp, or a host without ``/proc``, degrades to the
presence-only behaviour and never fails closed.
No control channel exists into a running gateway, so begin/cancel-drain writes
(or removes) ``{HERMES_HOME}/.drain_request.json`` and a gateway watcher reacts;
an ACTIVE marker means ``gateway_state -> "draining"``. Two lenient staleness
signals (either suffices): epoch mismatch (HERMES_HOME is a durable volume on
Hermes Cloud, so a marker survives the restart a drain-gated action ends in and
would park the fresh gateway in ``draining`` forever) and expiry (same-epoch
orphan past :data:`DRAIN_REQUEST_MAX_AGE_SECONDS`; re-writing refreshes it).
Reading never raises: a malformed file reads as ``{}`` — still drain-active
(fail-safe toward quiescing). Staleness rejects only on a *definite* verdict.
"""
from __future__ import annotations
import contextlib
import functools
import json
import logging
@@ -40,21 +20,18 @@ from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Optional
from gateway.memory_status import _parse_iso
from hermes_constants import get_hermes_home
from utils import atomic_json_write
import contextlib
_log = logging.getLogger(__name__)
_DRAIN_REQUEST_FILENAME = ".drain_request.json"
# Drain-gated lifecycle actions complete in minutes; an hour bounds the wedge a
# leaked marker can cause. Long drains refresh the marker instead of raising this.
# Drain-gated lifecycle actions complete in minutes; an hour bounds the wedge a leaked
# marker can cause. Long drains refresh the marker instead of raising this.
DRAIN_REQUEST_MAX_AGE_SECONDS = 3600.0
# Dedup for the expired-marker warning: the watcher re-reads every second and an
# expired orphan sits on disk until removed. Keyed by ``requested_at`` so a
# keep-alive re-write that later expires again logs again.
# Dedup for the expired-marker warning (the watcher re-reads every second); keyed by
# ``requested_at`` so a keep-alive re-write that later expires logs again.
_expiry_logged_for: Optional[str] = None
@@ -62,62 +39,40 @@ _expiry_logged_for: Optional[str] = None
def current_instantiation_epoch() -> str:
"""Identity of THIS container / VM instantiation ("<boot_id>:<pid1_start>").
Stable for the life of PID 1 (so an s6 respawn of just the gateway, or a
host ``hermes gateway restart`` under systemd/launchd, keeps honouring an
in-flight drain) but changes whenever the machine is recreated: boot_id
changes on a VM/microVM reboot, PID 1's start time on a plain
``docker restart`` (same host kernel, brand-new ``/init``).
Returns ``""`` when neither source is readable (non-Linux, no ``/proc``),
which disables the epoch check downstream — never fail-closed.
Stable for the life of PID 1 (a gateway-only respawn keeps honouring an
in-flight drain) but changes when the machine is recreated: boot_id on a VM
reboot, PID 1's start time on ``docker restart``. ``""`` when neither is
readable (non-Linux, no ``/proc``) disables the epoch check — never fail-closed.
"""
boot_id = ""
boot_id = pid1_start = ""
with contextlib.suppress(OSError):
boot_id = Path("/proc/sys/kernel/random/boot_id").read_text(encoding="utf-8").strip()
pid1_start = ""
try:
# "<pid> (<comm>) <state> ...": comm may contain spaces/parens, so split
# on the LAST ')'. starttime is field 22 (1-indexed) = tail index 19.
stat = Path("/proc/1/stat").read_text(encoding="utf-8")
pid1_start = stat.rsplit(")", 1)[1].split()[19]
except (OSError, IndexError):
pass
if not boot_id and not pid1_start:
return ""
return f"{boot_id}:{pid1_start}"
with contextlib.suppress(OSError, IndexError):
# "<pid> (<comm>) <state> ...": comm may contain spaces/parens, so split on the
# LAST ')'. starttime is field 22 (1-indexed) = tail index 19.
pid1_start = Path("/proc/1/stat").read_text(encoding="utf-8").rsplit(")", 1)[1].split()[19]
return f"{boot_id}:{pid1_start}" if (boot_id or pid1_start) else ""
def drain_request_path(home: Optional[Path] = None) -> Path:
"""Absolute path to the drain-request marker, respecting HERMES_HOME."""
base = home if home is not None else get_hermes_home()
return Path(base) / _DRAIN_REQUEST_FILENAME
return Path(home if home is not None else get_hermes_home()) / _DRAIN_REQUEST_FILENAME
def write_drain_request(
*,
principal: str = "drain-control",
suppress_notification: bool = False,
home: Optional[Path] = None,
*, principal: str = "drain-control", suppress_notification: bool = False, home: Optional[Path] = None
) -> dict[str, Any]:
"""Write the begin-drain marker atomically. Returns the payload written.
"""Write the begin-drain marker atomically; returns the payload.
Idempotent: re-writing refreshes ``requested_at`` (keep-alive past the
max-age). ``suppress_notification`` asks the shutdown that ends this drain
to skip ONLY the home-channel "gateway shutting down" broadcast (the
per-active-session interrupt ping is never suppressed); the policy of
which drains are quiet lives entirely in the caller. Defaults False so
legacy/operator drains behave exactly as before. The marker is stamped
with :func:`current_instantiation_epoch` so a copy surviving a machine
restart on the durable volume can be recognised as stale.
Re-writing refreshes ``requested_at`` (keep-alive past the max-age).
``suppress_notification`` skips ONLY the home-channel "gateway shutting down"
broadcast (the per-session interrupt ping is never suppressed); which drains
are quiet is the caller's policy. Stamped with the instantiation epoch so a
copy surviving a machine restart on the durable volume reads as stale.
"""
payload = {
"action": "drain",
"requested_at": datetime.now(timezone.utc).isoformat(),
"principal": principal,
"epoch": current_instantiation_epoch(),
"suppress_notification": bool(suppress_notification),
"action": "drain", "requested_at": datetime.now(timezone.utc).isoformat(), "principal": principal,
"epoch": current_instantiation_epoch(), "suppress_notification": bool(suppress_notification),
}
atomic_json_write(drain_request_path(home), payload)
return payload
@@ -129,65 +84,43 @@ def clear_drain_request(*, home: Optional[Path] = None) -> bool:
try:
path.unlink()
return True
except FileNotFoundError:
return False
except OSError as e:
_log.warning("drain-control: failed to remove %s: %s", path, e)
if not isinstance(e, FileNotFoundError):
_log.warning("drain-control: failed to remove %s: %s", path, e)
return False
def _marker_epoch_is_stale(body: dict[str, Any]) -> bool:
"""True iff ``body``'s epoch is a *definite* mismatch (both epochs known and differ)."""
current = current_instantiation_epoch()
marker_epoch = body.get("epoch")
return bool(current and marker_epoch and marker_epoch != current)
def _marker_is_expired(body: dict[str, Any]) -> bool:
"""True iff ``requested_at`` parses AND is older than the max-age.
Missing/unparseable timestamps and future-dated ones (clock skew) are
honoured. The expiry is logged once per marker, not per poll: this path
only fires when a writer leaked a marker, and the warning is the
operator's breadcrumb.
Missing/unparseable and future-dated (clock skew) timestamps are honoured.
Logged once per marker, not per poll — the operator's breadcrumb for a leak.
"""
global _expiry_logged_for
raw = body.get("requested_at")
if not isinstance(raw, str) or not raw:
requested_at = _parse_iso(raw)
if requested_at is None:
return False
try:
requested_at = datetime.fromisoformat(raw)
except ValueError:
return False
if requested_at.tzinfo is None:
requested_at = requested_at.replace(tzinfo=timezone.utc)
age = (datetime.now(timezone.utc) - requested_at).total_seconds()
if age <= DRAIN_REQUEST_MAX_AGE_SECONDS:
return False
if _expiry_logged_for != raw:
_expiry_logged_for = raw
_log.warning(
"drain-control: ignoring expired drain marker (requested_at=%s, "
"age=%.0fs > max %.0fs, principal=%s) — the drain that wrote it "
"was never cancelled; treating as stale so the gateway keeps "
"accepting turns.",
raw,
age,
DRAIN_REQUEST_MAX_AGE_SECONDS,
body.get("principal"),
"drain-control: ignoring expired drain marker (requested_at=%s, age=%.0fs > max %.0fs, principal=%s) "
"— the drain that wrote it was never cancelled; treating as stale so the gateway keeps accepting turns.",
raw, age, DRAIN_REQUEST_MAX_AGE_SECONDS, body.get("principal"),
)
return True
def _marker_is_stale(body: dict[str, Any]) -> bool:
"""True iff the marker is definitely from a drain that is already over."""
return _marker_epoch_is_stale(body) or _marker_is_expired(body)
def _active_drain_body(home: Optional[Path]) -> Optional[dict[str, Any]]:
"""Marker body if present AND not stale, else None."""
"""Marker body if present AND not stale (definite epoch mismatch or expired), else None."""
body = read_drain_request(home=home)
if body is None or _marker_is_stale(body):
if body is None:
return None
current, marker_epoch = current_instantiation_epoch(), body.get("epoch")
if (current and marker_epoch and marker_epoch != current) or _marker_is_expired(body):
return None
return body
@@ -198,12 +131,10 @@ def drain_requested(*, home: Optional[Path] = None) -> bool:
def drain_notification_suppressed(*, home: Optional[Path] = None) -> bool:
"""True iff an ACTIVE drain marker explicitly asks to suppress the shutdown broadcast.
"""True iff an ACTIVE marker asks to suppress the shutdown broadcast.
Uses the same activeness rule as :func:`drain_requested`, so an orphaned
marker can never silence a fresh gateway's legitimate broadcast. A legacy
marker without the field or a contentless ``{}`` body reads as False
(fail toward the louder behaviour).
Same activeness rule as :func:`drain_requested`, so an orphan can never silence
a fresh gateway; a marker without the field reads False (fail toward louder).
"""
body = _active_drain_body(home)
return bool(body and body.get("suppress_notification"))
@@ -214,10 +145,9 @@ def read_drain_request(*, home: Optional[Path] = None) -> Optional[dict[str, Any
path = drain_request_path(home)
try:
raw = path.read_text(encoding="utf-8")
except FileNotFoundError:
return None
except OSError as e:
_log.warning("drain-control: failed to read %s: %s", path, e)
if not isinstance(e, FileNotFoundError):
_log.warning("drain-control: failed to read %s: %s", path, e)
return None
try:
data = json.loads(raw)
gateway/hooks.py
+62 -96
View File
@@ -1,30 +1,18 @@
"""
Event Hook System
"""Event hook system: fires handlers at gateway lifecycle points.
Fires handlers at gateway lifecycle points. Hooks are discovered from
~/.hermes/hooks/<name>/ directories, each containing HOOK.yaml (name,
description, events) and handler.py (``def handle(event_type, context)``,
sync or async). Handler errors are logged and never block the pipeline.
Events:
gateway:startup, session:start, session:end (user ran /new or /reset),
session:reset, agent:start, agent:step (each tool-loop turn), agent:end,
command:* (any slash command; wildcard match).
Context passed to ``agent:start`` / ``agent:end``:
platform, user_id, chat_id, thread_id (Telegram forum-topic / thread root
id as string; empty when not in a thread), chat_type ("dm" | "group" |
"forum" | ""), session_id, message (truncated to 500 chars).
``agent:end`` adds: response (truncated to 500 chars), model, provider.
Handlers posting a follow-up into the same Telegram forum-topic should pass
``message_thread_id=int(thread_id)`` when ``chat_type == "forum"`` and
``thread_id`` is non-empty.
Hooks live in ~/.hermes/hooks/<name>/ with HOOK.yaml (name, description, events) and
handler.py (``def handle(event_type, context)``, sync or async); errors never block
the pipeline. Events: gateway:startup, session:start/end/reset, agent:start,
agent:step (each tool-loop turn), agent:end, command:* (wildcard). agent:* context:
platform, user_id, chat_id, thread_id ("" outside a thread), chat_type
("dm"|"group"|"forum"|""), session_id, message (500 chars); agent:end adds response,
model, provider. Forum follow-ups pass ``message_thread_id=int(thread_id)``.
"""
import asyncio
import importlib.util
import sys
from pathlib import Path
from typing import Any, Callable, Dict, List, Optional
import yaml
@@ -35,6 +23,42 @@ from hermes_cli.config import get_hermes_home
HOOKS_DIR = get_hermes_home() / "hooks"
def _skip(name: str, reason: str) -> None:
print(f"[hooks] Skipping {name}: {reason}", flush=True)
def _load_hook_dir(hook_dir: Path) -> Optional[tuple]:
"""``(name, events, handle_fn, description)`` for a valid hook dir, else None (reason printed)."""
manifest_path, handler_path = hook_dir / "HOOK.yaml", hook_dir / "handler.py"
if not manifest_path.exists() or not handler_path.exists():
return None
manifest = yaml.safe_load(manifest_path.read_text(encoding="utf-8"))
if not manifest or not isinstance(manifest, dict):
return _skip(hook_dir.name, "invalid HOOK.yaml")
hook_name = manifest.get("name", hook_dir.name)
events = manifest.get("events", [])
if not events:
return _skip(hook_name, "no events declared")
# Register in sys.modules BEFORE exec_module so Pydantic/dataclass forward references
# (``from __future__ import annotations``) resolve; otherwise a handler declaring a
# BaseModel fails at first dispatch with "TypeAdapter ... is not fully defined".
module_name = f"hermes_hook_{hook_name}"
spec = importlib.util.spec_from_file_location(module_name, handler_path)
if spec is None or spec.loader is None:
return _skip(hook_name, "could not load handler.py")
module = importlib.util.module_from_spec(spec)
sys.modules[module_name] = module
try:
spec.loader.exec_module(module)
except Exception:
sys.modules.pop(module_name, None)
raise
handle_fn = getattr(module, "handle", None)
if handle_fn is None:
return _skip(hook_name, "no 'handle' function found")
return hook_name, events, handle_fn, manifest.get("description", "")
class HookRegistry:
"""Discovers, loads, and fires event hooks."""
@@ -48,81 +72,33 @@ class HookRegistry:
def _register_builtin_hooks(self) -> None:
"""Extension point for always-on built-in hooks; currently none shipped."""
return
def discover_and_load(self) -> None:
"""Register built-in hooks, then load every valid hook dir under HOOKS_DIR."""
self._register_builtin_hooks()
if not HOOKS_DIR.exists():
return
for hook_dir in sorted(HOOKS_DIR.iterdir()):
if not hook_dir.is_dir():
continue
manifest_path = hook_dir / "HOOK.yaml"
handler_path = hook_dir / "handler.py"
if not manifest_path.exists() or not handler_path.exists():
continue
try:
manifest = yaml.safe_load(manifest_path.read_text(encoding="utf-8"))
if not manifest or not isinstance(manifest, dict):
print(f"[hooks] Skipping {hook_dir.name}: invalid HOOK.yaml", flush=True)
continue
hook_name = manifest.get("name", hook_dir.name)
events = manifest.get("events", [])
if not events:
print(f"[hooks] Skipping {hook_name}: no events declared", flush=True)
continue
# Register in sys.modules BEFORE exec_module so Pydantic/dataclass
# forward references (from ``from __future__ import annotations``)
# resolve; otherwise a handler declaring a BaseModel fails at first
# dispatch with "TypeAdapter ... is not fully defined".
module_name = f"hermes_hook_{hook_name}"
spec = importlib.util.spec_from_file_location(module_name, handler_path)
if spec is None or spec.loader is None:
print(f"[hooks] Skipping {hook_name}: could not load handler.py", flush=True)
continue
module = importlib.util.module_from_spec(spec)
sys.modules[module_name] = module
try:
spec.loader.exec_module(module)
except Exception:
sys.modules.pop(module_name, None)
raise
handle_fn = getattr(module, "handle", None)
if handle_fn is None:
print(f"[hooks] Skipping {hook_name}: no 'handle' function found", flush=True)
continue
for event in events:
self._handlers.setdefault(event, []).append(handle_fn)
self._loaded_hooks.append({
"name": hook_name,
"description": manifest.get("description", ""),
"events": events,
"path": str(hook_dir),
})
print(f"[hooks] Loaded hook '{hook_name}' for events: {events}", flush=True)
loaded = _load_hook_dir(hook_dir)
except Exception as e:
print(f"[hooks] Error loading hook {hook_dir.name}: {e}", flush=True)
continue
if loaded is None:
continue
hook_name, events, handle_fn, description = loaded
for event in events:
self._handlers.setdefault(event, []).append(handle_fn)
self._loaded_hooks.append(
{"name": hook_name, "description": description, "events": events, "path": str(hook_dir)}
)
print(f"[hooks] Loaded hook '{hook_name}' for events: {events}", flush=True)
def _resolve_handlers(self, event_type: str) -> List[Callable]:
"""Exact-match handlers first, then ``<base>:*`` wildcard handlers.
A handler registered for a bare base type ("agent") does NOT fire for
"agent:start" — only exact matches and explicit wildcards.
"""
"""Exact-match handlers first, then ``<base>:*`` wildcards. A bare base type
("agent") does NOT fire for "agent:start" — only exact matches and explicit wildcards."""
handlers = list(self._handlers.get(event_type, []))
if ":" in event_type:
handlers.extend(self._handlers.get(f"{event_type.split(':')[0]}:*", []))
@@ -132,26 +108,16 @@ class HookRegistry:
"""Fire all handlers for an event, discarding return values."""
await self.emit_collect(event_type, context)
async def emit_collect(
self,
event_type: str,
context: Optional[Dict[str, Any]] = None,
) -> List[Any]:
"""Fire handlers and return their non-None return values in order.
Used for decision-style hooks (e.g. ``command:<name>`` policies that
allow/deny/rewrite a command). A failing handler is logged and does not
abort the remaining handlers.
"""
async def emit_collect(self, event_type: str, context: Optional[Dict[str, Any]] = None) -> List[Any]:
"""Fire handlers and return their non-None return values in order (decision-style
hooks, e.g. ``command:<name>`` policies). A failing handler is logged, not fatal."""
if context is None:
context = {}
results: List[Any] = []
for fn in self._resolve_handlers(event_type):
try:
result = fn(event_type, context)
if asyncio.iscoroutine(result): # sync and async handlers both supported
result = await result
result = await result if asyncio.iscoroutine(result) else result # sync or async handlers
if result is not None:
results.append(result)
except Exception as e:
gateway/hosted_room_discussion.py
+127 -275
View File
@@ -1,20 +1,15 @@
"""Deterministic policy for same-gateway hosted-room Discussions.
Translates a frozen local member roster plus the complete typed room log into
one next driver task. Pure: no I/O, no workers, no transport or model
knowledge. Callers persist the task with :mod:`gateway.hosted_room_driver`
and append publication plans with :mod:`gateway.hosted_rooms`.
The driver payload is deliberately not widened: Discussion coordinates live
in deterministic ``TaskIdentity`` values and typed terminal events, so a
restart can reconstruct a task from durable state. Callers must reconcile
terminal driver rows into publication plans before asking for the next task.
Pure (no I/O, transport or model knowledge): a frozen local roster plus the complete
typed room log yields at most one next driver task. Discussion coordinates live in
deterministic ``TaskIdentity`` values and typed terminal events rather than a widened
driver payload, so a restart reconstructs tasks from durable state. Callers must
reconcile terminal driver rows into publication plans before asking for the next task.
"""
from __future__ import annotations
import hashlib
import json
import re
from collections.abc import Iterable, Mapping, Sequence
from dataclasses import dataclass
@@ -24,6 +19,7 @@ from typing import Any, Literal
from gateway import hosted_room_driver as driver
from gateway import hosted_rooms
from gateway import hosted_rooms_common as common
from gateway.hosted_rooms_common import compact_json
MAX_DISCUSSION_MEMBERS = 6
@@ -42,29 +38,22 @@ _MENTION_RE = re.compile(r"@([A-Za-z0-9][A-Za-z0-9._:-]*)", re.IGNORECASE)
_TURN_ID_RE = re.compile(
r"^d(?P<source>[1-9][0-9]*)\.r(?P<round>[0-2])\."
r"p(?P<position>[0-5])\.s(?P<seen>[1-9][0-9]*)\."
r"m(?P<member>[0-9a-f]{24})$"
)
r"m(?P<member>[0-9a-f]{24})$")
_LOCAL_TARGET_FIELDS = frozenset({"kind", "profile"})
_PEER_TARGET_FIELDS = frozenset({"kind", "peer_id", "installation_id", "profile", "capability_digest"})
_REMOTE_MEMBER_FIELDS = frozenset({
"connectionId", "connectionKind", "connectionLabel", "connection_id",
"connection_kind", "connection_label", "remoteSource", "route",
"sourceMissing", "sourceReachable", "sourceScoped", "targetProfile",
"target_profile",
"connectionId", "connectionKind", "connectionLabel", "connection_id", "connection_kind", "connection_label",
"remoteSource", "route", "sourceMissing", "sourceReachable", "sourceScoped", "targetProfile", "target_profile",
})
_USER_PAYLOAD_FIELDS = frozenset({"text", "thread_id"})
_TURN_COORDINATE_FIELDS = frozenset({
"discussion_event_id", "member_id", "member_index", "round_index",
"task_id", "thread_id", "turn_id",
})
_TURN_COORDINATE_FIELDS = frozenset(
{"discussion_event_id", "member_id", "member_index", "round_index", "task_id", "thread_id", "turn_id", })
_MEMBER_MESSAGE_FIELDS = _TURN_COORDINATE_FIELDS | {"text"}
_TERMINAL_COMMON_FIELDS = _TURN_COORDINATE_FIELDS | {"seen_through_seq"}
_TERMINAL_EXTRA_FIELDS = {
"turn.settled": frozenset({"message_event_id", "passed"}),
"turn.failed": frozenset({"error"}),
"turn.cancelled": frozenset({"reason"}),
"turn.deferred": frozenset({"execution_generation", "reason"}),
"turn.settled": frozenset({"message_event_id", "passed"}), "turn.failed": frozenset({"error"}),
"turn.cancelled": frozenset({"reason"}), "turn.deferred": frozenset({"execution_generation", "reason"}),
}
_TERMINAL_OPTIONAL_FIELDS = {"turn.failed": frozenset({"reason_code"})}
_TERMINAL_EVENT_KINDS = frozenset(_TERMINAL_EXTRA_FIELDS)
@@ -72,15 +61,12 @@ _TERMINAL_EVENT_KINDS = frozenset(_TERMINAL_EXTRA_FIELDS)
_GATEWAY_EVENT_FIELDS = {
"room.activity": (
frozenset({"status", "reason_code", "thread_id", "discussion_event_id"}),
("reason_code", "thread_id", "discussion_event_id"),
),
"room.stop_requested": (frozenset({"cancel_id"}), ("cancel_id",)),
}
("reason_code", "thread_id", "discussion_event_id")),
"room.stop_requested": (frozenset({"cancel_id"}), ("cancel_id",))}
_EPOCH_STAMPED_KINDS = _TERMINAL_EVENT_KINDS | {"message.member", *_GATEWAY_EVENT_FIELDS}
class DiscussionPolicyError(ValueError):
"""Base class for invalid policy input or unreconstructable state."""
class DiscussionPolicyError(ValueError): """Base class for invalid policy input or unreconstructable state."""
class DiscussionValidationError(DiscussionPolicyError):
@@ -155,8 +141,7 @@ class EventPlan:
"room_id": room_id, "event_id": self.event_id, "kind": self.kind,
"actor": dict(self.actor), "payload": dict(self.payload),
"authority_gateway_id": self.authority_gateway_id,
"authority_epoch": self.authority_epoch,
}
"authority_epoch": self.authority_epoch}
@dataclass(frozen=True)
@@ -178,22 +163,20 @@ class _ValidatedEvent:
payload: Mapping[str, Any]
_identifier = partial(
common.identifier, error=DiscussionValidationError, max_chars=driver.MAX_IDENTIFIER_CHARS
)
_identifier = partial(common.identifier, error=DiscussionValidationError, max_chars=driver.MAX_IDENTIFIER_CHARS)
_exact_fields = partial(common.exact_fields, error=DiscussionValidationError)
_bounded_int = partial(common.bounded_int, error=DiscussionValidationError)
def _positive_int(value: Any, *, label: str) -> int:
return common.positive_int(
value, error=DiscussionValidationError, message=f"{label} must be a positive integer"
)
return _bounded_int(value, message=f"{label} must be a positive integer", low=1)
def _zero_based_int(value: Any, *, label: str, maximum: int) -> int:
if isinstance(value, bool) or not isinstance(value, int) or not 0 <= value <= maximum:
raise DiscussionValidationError(f"{label} must be an integer between 0 and {maximum}")
return value
message = f"{label} must be an integer between 0 and {maximum}"
return _bounded_int(value, message=message, high=maximum)
_text = partial(common.text, error=DiscussionValidationError)
def _peer_target(member: DiscussionMember) -> Mapping[str, Any] | None:
@@ -204,25 +187,15 @@ def _peer_target(member: DiscussionMember) -> Mapping[str, Any] | None:
def validate_user_payload(value: Any) -> dict[str, Any]:
"""Validate and normalize the exact ``message.user`` Discussion payload."""
payload = _exact_fields(value, label="user payload", required=_USER_PAYLOAD_FIELDS)
text = payload["text"]
if not isinstance(text, str):
raise DiscussionValidationError("user payload text must be a string")
text = text.strip()
if not text:
raise DiscussionValidationError("user payload text must not be empty")
if len(text.encode("utf-8")) > MAX_USER_TEXT_BYTES:
raise DiscussionValidationError("user payload text is too large")
text = _text(payload["text"], label="user payload text", max_bytes=MAX_USER_TEXT_BYTES)
return {"text": text, "thread_id": _identifier(payload["thread_id"], label="thread_id")}
def _validate_member_target(
value: Any, *, profile: str, known_profiles: set[str], index: int
) -> dict[str, Any]:
def _validate_member_target(value: Any, *, profile: str, known_profiles: set[str], index: int) -> dict[str, Any]:
if value is None:
if profile not in known_profiles:
raise DiscussionValidationError(
f"member {index} profile '{profile}' is not local to this gateway"
)
f"member {index} profile '{profile}' is not local to this gateway")
return {"kind": "local", "profile": profile}
if not isinstance(value, Mapping):
raise DiscussionValidationError(f"member {index} target must be an object")
@@ -231,19 +204,16 @@ def _validate_member_target(
raise DiscussionValidationError(f"member {index} target kind must be local or peer")
target = _exact_fields(
value, label=f"member {index} {kind} target",
required=_LOCAL_TARGET_FIELDS if kind == "local" else _PEER_TARGET_FIELDS,
)
required=_LOCAL_TARGET_FIELDS if kind == "local" else _PEER_TARGET_FIELDS)
target_profile = _identifier(target["profile"], label=f"member {index} target profile")
if kind == "local":
if target_profile != profile or profile not in known_profiles:
raise DiscussionValidationError(
f"member {index} local target does not match a local profile"
)
f"member {index} local target does not match a local profile")
return {"kind": "local", "profile": profile}
if target_profile != profile:
raise DiscussionValidationError(
f"member {index} peer target profile does not match member profile"
)
f"member {index} peer target profile does not match member profile")
capability_digest = target["capability_digest"]
if not isinstance(capability_digest, str) or not re.fullmatch(r"[0-9a-f]{64}", capability_digest):
raise DiscussionValidationError(f"member {index} capability_digest must be a sha256 digest")
@@ -252,8 +222,7 @@ def _validate_member_target(
"peer_id": _identifier(target["peer_id"], label=f"member {index} peer_id"),
"installation_id": _identifier(target["installation_id"], label=f"member {index} installation_id"),
"profile": target_profile,
"capability_digest": capability_digest,
}
"capability_digest": capability_digest}
def validate_roster(value: Any, *, local_profiles: Iterable[str]) -> tuple[DiscussionMember, ...]:
@@ -263,8 +232,7 @@ def validate_roster(value: Any, *, local_profiles: Iterable[str]) -> tuple[Discu
if not MIN_DISCUSSION_MEMBERS <= len(value) <= MAX_DISCUSSION_MEMBERS:
raise DiscussionValidationError(
f"members must contain between {MIN_DISCUSSION_MEMBERS} and "
f"{MAX_DISCUSSION_MEMBERS} entries"
)
f"{MAX_DISCUSSION_MEMBERS} entries")
known_profiles = {_identifier(profile, label="local profile") for profile in local_profiles}
members: list[DiscussionMember] = []
@@ -278,19 +246,16 @@ def validate_roster(value: Any, *, local_profiles: Iterable[str]) -> tuple[Discu
remote_fields = frozenset(raw) & _REMOTE_MEMBER_FIELDS
if remote_fields:
raise DiscussionValidationError(
f"member {index} contains cross-gateway fields: {', '.join(sorted(remote_fields))}"
)
f"member {index} contains cross-gateway fields: {', '.join(sorted(remote_fields))}")
member = _exact_fields(
raw, label=f"member {index}",
required=frozenset({"member_id", "profile", "handle"}),
optional=frozenset({"display_name", "target"}),
)
optional=frozenset({"display_name", "target"}))
member_id = _identifier(member["member_id"], label=f"member {index} id")
profile = _identifier(member["profile"], label=f"member {index} profile")
handle = _identifier(member["handle"], label=f"member {index} handle")
target = _validate_member_target(
member.get("target"), profile=profile, known_profiles=known_profiles, index=index
)
member.get("target"), profile=profile, known_profiles=known_profiles, index=index)
display_name = member.get("display_name", "")
if not isinstance(display_name, str):
raise DiscussionValidationError(f"member {index} display_name must be a string")
@@ -298,24 +263,20 @@ def validate_roster(value: Any, *, local_profiles: Iterable[str]) -> tuple[Discu
if len(display_name) > hosted_rooms.MAX_ACTOR_LABEL_CHARS:
raise DiscussionValidationError(f"member {index} display_name is too long")
target_key = json.dumps(target, sort_keys=True, separators=(",", ":")).casefold()
target_key = compact_json(target, ensure_ascii=False).casefold()
target_message = (
"member profiles must be unique" if target.get("kind") == "local"
else "member targets must be unique"
)
else "member targets must be unique")
for key, seen, message in (
(target_key, targets, target_message),
(handle.casefold(), handles,
"member handles must be unique and cannot reserve @all or @everyone"),
(member_id.casefold(), member_ids, "member ids must be unique"),
):
(handle.casefold(), handles, "member handles must be unique and cannot reserve @all or @everyone"),
(member_id.casefold(), member_ids, "member ids must be unique")):
if key in seen:
raise DiscussionValidationError(message)
seen.add(key)
members.append(DiscussionMember(
member_id=member_id, profile=profile, handle=handle,
display_name=display_name, target=target,
))
display_name=display_name, target=target))
return tuple(members)
@@ -336,9 +297,7 @@ def validate_room(value: Any, *, local_profiles: Iterable[str]) -> DiscussionRoo
authority_epoch = _positive_int(value.get("authority_epoch"), label="authority_epoch")
members = validate_roster(value.get("members"), local_profiles=local_profiles)
return DiscussionRoom(
room_id=room_id, name=name, members=members,
gateway_id=gateway_id, authority_epoch=authority_epoch,
)
room_id=room_id, name=name, members=members, gateway_id=gateway_id, authority_epoch=authority_epoch)
def is_pass_text(value: Any) -> bool:
@@ -367,8 +326,7 @@ def resolve_mentions(
def _unaddressed_member_mentions(
messages: Sequence[_ValidatedEvent], room: DiscussionRoom
) -> tuple[DiscussionMember, ...]:
messages: Sequence[_ValidatedEvent], room: DiscussionRoom) -> tuple[DiscussionMember, ...]:
"""Return peers explicitly cited by a Bot and not heard from afterward."""
cited_at: dict[str, int] = {}
last_post_at: dict[str, int] = {}
@@ -384,8 +342,7 @@ def _unaddressed_member_mentions(
return tuple(
member for member in room.members
if member.member_id in cited_at
and last_post_at.get(member.member_id, 0) <= cited_at[member.member_id]
)
and last_post_at.get(member.member_id, 0) <= cited_at[member.member_id])
def _require_gateway_actor(actor: Mapping[str, Any], room: DiscussionRoom, message: str) -> None:
@@ -395,10 +352,10 @@ def _require_gateway_actor(actor: Mapping[str, Any], room: DiscussionRoom, messa
def _member_by_id(room: DiscussionRoom, member_id: Any) -> DiscussionMember:
normalized = _identifier(member_id, label="member_id")
for member in room.members:
if member.member_id == normalized:
return member
raise DiscussionValidationError(f"unknown Discussion member '{normalized}'")
member = next((m for m in room.members if m.member_id == normalized), None)
if member is None:
raise DiscussionValidationError(f"unknown Discussion member '{normalized}'")
return member
def _validate_turn_coordinates(payload: Mapping[str, Any], room: DiscussionRoom) -> None:
@@ -414,8 +371,7 @@ def _validate_turn_coordinates(payload: Mapping[str, Any], room: DiscussionRoom)
def _validate_user_event(
kind: str, payload: Mapping[str, Any], actor: Mapping[str, Any], room: DiscussionRoom
) -> Mapping[str, Any]:
kind: str, payload: Mapping[str, Any], actor: Mapping[str, Any], room: DiscussionRoom) -> Mapping[str, Any]:
payload = validate_user_payload(payload)
if actor.get("kind") != "user":
raise DiscussionValidationError("message.user requires a user actor")
@@ -423,8 +379,7 @@ def _validate_user_event(
def _validate_member_message(
kind: str, payload: Mapping[str, Any], actor: Mapping[str, Any], room: DiscussionRoom
) -> Mapping[str, Any]:
kind: str, payload: Mapping[str, Any], actor: Mapping[str, Any], room: DiscussionRoom) -> Mapping[str, Any]:
_exact_fields(payload, label="message.member payload", required=_MEMBER_MESSAGE_FIELDS)
_validate_turn_coordinates(payload, room)
text = payload.get("text")
@@ -433,9 +388,7 @@ def _validate_member_message(
member = _member_by_id(room, payload.get("member_id"))
peer = _peer_target(member)
expected = {
"kind": "member",
"id": member.member_id,
"profile": member.profile,
"kind": "member", "id": member.member_id, "profile": member.profile,
"connection_id": peer.get("peer_id") if peer else None,
}
if any(actor.get(key) != value for key, value in expected.items()):
@@ -444,13 +397,11 @@ def _validate_member_message(
def _validate_terminal_event(
kind: str, payload: Mapping[str, Any], actor: Mapping[str, Any], room: DiscussionRoom
) -> Mapping[str, Any]:
kind: str, payload: Mapping[str, Any], actor: Mapping[str, Any], room: DiscussionRoom) -> Mapping[str, Any]:
_exact_fields(
payload, label=f"{kind} payload",
required=_TERMINAL_COMMON_FIELDS | _TERMINAL_EXTRA_FIELDS[kind],
optional=_TERMINAL_OPTIONAL_FIELDS.get(kind, frozenset()),
)
optional=_TERMINAL_OPTIONAL_FIELDS.get(kind, frozenset()))
_validate_turn_coordinates(payload, room)
_positive_int(payload.get("seen_through_seq"), label="seen_through_seq")
if actor.get("connection_id") is not None:
@@ -474,15 +425,12 @@ def _validate_terminal_event(
from tools.bot_failure_reasons import ALL_REASONS
if payload["reason_code"] not in ALL_REASONS:
raise DiscussionValidationError(
"turn.failed reason_code must use the shared failure vocabulary"
)
raise DiscussionValidationError("turn.failed reason_code must use the shared failure vocabulary")
return payload
def _validate_gateway_event(
kind: str, payload: Mapping[str, Any], actor: Mapping[str, Any], room: DiscussionRoom
) -> Mapping[str, Any]:
kind: str, payload: Mapping[str, Any], actor: Mapping[str, Any], room: DiscussionRoom) -> Mapping[str, Any]:
fields, identifier_fields = _GATEWAY_EVENT_FIELDS[kind]
_exact_fields(payload, label=f"{kind} payload", required=fields)
if kind == "room.activity" and payload.get("status") not in {"settled", "bounded"}:
@@ -497,8 +445,7 @@ _EVENT_VALIDATORS = {
"message.user": _validate_user_event,
"message.member": _validate_member_message,
**dict.fromkeys(_TERMINAL_EVENT_KINDS, _validate_terminal_event),
**dict.fromkeys(_GATEWAY_EVENT_FIELDS, _validate_gateway_event),
}
**dict.fromkeys(_GATEWAY_EVENT_FIELDS, _validate_gateway_event)}
def _validate_event(raw: Any, *, room: DiscussionRoom, previous_seq: int) -> _ValidatedEvent:
@@ -514,8 +461,7 @@ def _validate_event(raw: Any, *, room: DiscussionRoom, previous_seq: int) -> _Va
for value, expected, message in (
(kind, str, "event kind must be a string"),
(actor, Mapping, "event actor must be an object"),
(payload, Mapping, "event payload must be an object"),
):
(payload, Mapping, "event payload must be an object")):
if not isinstance(value, expected):
raise DiscussionValidationError(message)
if kind in _EPOCH_STAMPED_KINDS and raw.get("authority_epoch") != room.authority_epoch:
@@ -526,9 +472,7 @@ def _validate_event(raw: Any, *, room: DiscussionRoom, previous_seq: int) -> _Va
return _ValidatedEvent(raw=raw, seq=seq, event_id=event_id, kind=kind, actor=actor, payload=payload)
def _validated_events(
events: Sequence[Mapping[str, Any]], *, room: DiscussionRoom
) -> tuple[_ValidatedEvent, ...]:
def _validated_events(events: Sequence[Mapping[str, Any]], *, room: DiscussionRoom) -> tuple[_ValidatedEvent, ...]:
validated: list[_ValidatedEvent] = []
previous_seq = 0
event_ids: set[str] = set()
@@ -562,14 +506,12 @@ def _derive_member_watermarks(events: Sequence[_ValidatedEvent]) -> dict[tuple[s
if previous is not None:
if previous.kind != "turn.deferred":
raise DiscussionValidationError(
f"task '{task_id}' has more than one terminal room event"
)
f"task '{task_id}' has more than one terminal room event")
if event.kind == "turn.deferred" and int(
event.payload["execution_generation"]
) <= int(previous.payload["execution_generation"]):
raise DiscussionValidationError(
f"task '{task_id}' deferral generation did not advance"
)
f"task '{task_id}' deferral generation did not advance")
terminal_by_task[task_id] = event
key = (str(event.payload["thread_id"]), str(event.payload["member_id"]))
watermark = int(event.payload["seen_through_seq"])
@@ -577,8 +519,7 @@ def _derive_member_watermarks(events: Sequence[_ValidatedEvent]) -> dict[tuple[s
message = messages_by_id.get(str(event.payload["message_event_id"]))
if message is None or any(
message.payload.get(field) != event.payload.get(field)
for field in ("task_id", "member_id", "thread_id")
):
for field in ("task_id", "member_id", "thread_id")):
raise DiscussionValidationError("turn.settled references no matching member message")
watermark = max(watermark, message.seq)
watermarks[key] = max(watermarks.get(key, 0), watermark)
@@ -586,10 +527,7 @@ def _derive_member_watermarks(events: Sequence[_ValidatedEvent]) -> dict[tuple[s
def _member_digest(member: DiscussionMember) -> str:
target = json.dumps(
member.target or {"kind": "local", "profile": member.profile},
sort_keys=True, separators=(",", ":"),
)
target = compact_json(member.target or {"kind": "local", "profile": member.profile}, ensure_ascii=False)
seed = f"{member.member_id}\0{member.profile}\0{member.handle}\0{target}"
return hashlib.sha256(seed.encode("utf-8")).hexdigest()[:24]
@@ -626,28 +564,22 @@ def _truncate_utf8_text(value: Any, *, max_bytes: int, suffix: str = "") -> str:
def _build_prompt(
*, room: DiscussionRoom, member: DiscussionMember, messages: Sequence[_ValidatedEvent],
watermark: int, seen_through_seq: int,
) -> str:
delta = [event for event in messages if watermark < event.seq <= seen_through_seq][
-MAX_DISCUSSION_DELTA_LINES:
]
watermark: int, seen_through_seq: int) -> str:
delta = [event for event in messages if watermark < event.seq <= seen_through_seq][- MAX_DISCUSSION_DELTA_LINES:]
peers = ", ".join(
f"@{candidate.handle}" for candidate in room.members if candidate.member_id != member.member_id
)
f"@{candidate.handle}" for candidate in room.members if candidate.member_id != member.member_id)
opening = [
f'[Discussion: "{room.name}"] You are @{member.handle}, one participant '
f"with {peers or 'no other members'} and the user.",
"",
"New messages in this thread since your last turn (oldest first):",
]
"New messages in this thread since your last turn (oldest first):"]
rules = [
"",
"Rules for this Discussion:",
"- Reply with one conversational message only when you have something new worth adding.",
'- If you have nothing new to add, reply with exactly "(pass)".',
"- Mention a teammate by handle to pull them into the next round; do not repeat points already made.",
"- Never reveal content from private conversations. Your reply is published verbatim.",
]
"- Never reveal content from private conversations. Your reply is published verbatim."]
fixed_bytes = len("\n".join([*opening, *rules]).encode("utf-8"))
available = max(0, driver.MAX_PROMPT_BYTES - fixed_bytes - 1)
selected: list[str] = []
@@ -674,54 +606,36 @@ def _build_prompt(
def _make_task_plan(
*, room: DiscussionRoom, discussion_event: _ValidatedEvent, member: DiscussionMember,
member_index: int, round_index: int, seen_through_seq: int, prompt: str,
) -> DiscussionTaskPlan:
member_index: int, round_index: int, seen_through_seq: int, prompt: str) -> DiscussionTaskPlan:
turn_id = (
f"d{discussion_event.seq}.r{round_index}.p{member_index}."
f"s{seen_through_seq}.m{_member_digest(member)}"
)
seed = json.dumps(
{
"discussion_event_id": discussion_event.event_id,
"member_id": member.member_id,
"member_index": member_index,
"prompt_sha256": hashlib.sha256(prompt.encode("utf-8")).hexdigest(),
"room_id": room.room_id,
"round_index": round_index,
"seen_through_seq": seen_through_seq,
"source_event_seq": discussion_event.seq,
"thread_id": discussion_event.payload["thread_id"],
},
ensure_ascii=True, sort_keys=True, separators=(",", ":"),
)
f"s{seen_through_seq}.m{_member_digest(member)}")
seed = compact_json({
"discussion_event_id": discussion_event.event_id, "member_id": member.member_id, "member_index": member_index,
"prompt_sha256": hashlib.sha256(prompt.encode("utf-8")).hexdigest(), "room_id": room.room_id,
"round_index": round_index, "seen_through_seq": seen_through_seq, "source_event_seq": discussion_event.seq,
"thread_id": discussion_event.payload["thread_id"],
})
task_id = f"dtask:{hashlib.sha256(seed.encode('utf-8')).hexdigest()[:48]}"
identity = driver.TaskIdentity(
room_id=room.room_id, task_id=task_id,
thread_id=str(discussion_event.payload["thread_id"]), turn_id=turn_id,
)
thread_id=str(discussion_event.payload["thread_id"]), turn_id=turn_id)
payload = {
"target_member_id": member.member_id,
"target_profile": member.profile,
"prompt": prompt,
"target_member_id": member.member_id, "target_profile": member.profile, "prompt": prompt,
"source_event_seq": discussion_event.seq,
}
return DiscussionTaskPlan(
identity=identity, payload=payload, discussion_event_id=discussion_event.event_id,
member=member, member_index=member_index, round_index=round_index,
seen_through_seq=seen_through_seq,
)
identity=identity, payload=payload, discussion_event_id=discussion_event.event_id, member=member,
member_index=member_index, round_index=round_index, seen_through_seq=seen_through_seq)
def _pending_discussion(validated: Sequence[_ValidatedEvent]) -> _ValidatedEvent | None:
"""Oldest latest-per-thread user message not stopped and not yet completed."""
stopped_through_seq = max(
(event.seq for event in validated if event.kind == "room.stop_requested"), default=0
)
stopped_through_seq = max((event.seq for event in validated if event.kind == "room.stop_requested"), default=0)
completed_discussion_ids = {
str(event.payload["discussion_event_id"])
for event in validated
if event.kind == "room.activity" and event.payload.get("status") in {"settled", "bounded"}
}
if event.kind == "room.activity" and event.payload.get("status") in {"settled", "bounded"}}
latest_by_thread: dict[str, _ValidatedEvent] = {}
for event in validated:
if event.kind == "message.user":
@@ -729,8 +643,7 @@ def _pending_discussion(validated: Sequence[_ValidatedEvent]) -> _ValidatedEvent
pending = [
event
for event in sorted(latest_by_thread.values(), key=lambda item: item.seq)
if event.seq > stopped_through_seq and event.event_id not in completed_discussion_ids
]
if event.seq > stopped_through_seq and event.event_id not in completed_discussion_ids]
return pending[0] if pending else None
@@ -742,8 +655,7 @@ def _thread_messages(
committed_member_message_ids = {
str(event.payload["message_event_id"])
for event in validated
if event.kind == "turn.settled" and event.payload.get("message_event_id") is not None
}
if event.kind == "turn.settled" and event.payload.get("message_event_id") is not None}
# Publication writes the visible member message before the terminal event.
# A crash in that gap leaves the message in the log, but it is not committed
# policy input yet: ignoring it reproduces the original task coordinates so
@@ -754,15 +666,12 @@ def _thread_messages(
if event.payload.get("thread_id") == thread_id
and (
event.kind == "message.user"
or (event.kind == "message.member" and event.event_id in committed_member_message_ids)
)
)
or (event.kind == "message.member" and event.event_id in committed_member_message_ids)))
discussion_messages = tuple(event for event in thread_messages if event.seq >= discussion.seq)
member_messages = tuple(
event for event in thread_messages
if event.kind == "message.member"
and event.payload.get("discussion_event_id") == discussion.event_id
)
and event.payload.get("discussion_event_id") == discussion.event_id)
return thread_messages, discussion_messages, member_messages
@@ -772,20 +681,15 @@ def _effective_watermarks(
watermarks = {
(str(thread_id), str(member_id)): int(value)
for (thread_id, member_id), value in (initial_watermarks or {}).items()
if int(value) >= 0
}
if int(value) >= 0}
for key, value in _derive_member_watermarks(validated).items():
watermarks[key] = max(watermarks.get(key, 0), value)
return watermarks
def plan_next_task(
room_value: Any,
events: Sequence[Mapping[str, Any]],
*,
local_profiles: Iterable[str],
initial_watermarks: Mapping[tuple[str, str], int] | None = None,
) -> DiscussionDecision:
room_value: Any, events: Sequence[Mapping[str, Any]], *, local_profiles: Iterable[str],
initial_watermarks: Mapping[tuple[str, str], int] | None = None) -> DiscussionDecision:
"""Replay the complete room log and return at most one next member task."""
room = validate_room(room_value, local_profiles=local_profiles)
validated = _validated_events(events, room=room)
@@ -795,8 +699,7 @@ def plan_next_task(
thread_id = str(discussion.payload["thread_id"])
decide = partial(
DiscussionDecision, discussion_event_id=discussion.event_id,
source_event_seq=discussion.seq, thread_id=thread_id,
)
source_event_seq=discussion.seq, thread_id=thread_id)
thread_messages, discussion_messages, member_messages = _thread_messages(validated, discussion)
if len(member_messages) >= MAX_DISCUSSION_MESSAGES:
@@ -806,8 +709,7 @@ def plan_next_task(
(int(event.payload["round_index"]), str(event.payload["member_id"])): event
for event in validated
if event.kind in _TERMINAL_EVENT_KINDS
and event.payload.get("discussion_event_id") == discussion.event_id
}
and event.payload.get("discussion_event_id") == discussion.event_id}
watermarks = _effective_watermarks(validated, initial_watermarks)
seen_through_seq = max(event.seq for event in thread_messages)
@@ -820,8 +722,7 @@ def plan_next_task(
responders = (
resolve_mentions((str(discussion.payload["text"]),), room.members)
if round_index == 0
else _unaddressed_member_mentions(discussion_messages, room)
)
else _unaddressed_member_mentions(discussion_messages, room))
for member_index, member in enumerate(_rotate(responders, round_index)):
if (round_index, member.member_id) in terminals:
continue
@@ -830,12 +731,10 @@ def plan_next_task(
continue
prompt = _build_prompt(
room=room, member=member, messages=thread_messages,
watermark=watermark, seen_through_seq=seen_through_seq,
)
watermark=watermark, seen_through_seq=seen_through_seq)
task = _make_task_plan(
room=room, discussion_event=discussion, member=member, member_index=member_index,
round_index=round_index, seen_through_seq=seen_through_seq, prompt=prompt,
)
round_index=round_index, seen_through_seq=seen_through_seq, prompt=prompt)
return decide("task", "member_turn", task=task)
if not any(int(event.payload["round_index"]) == round_index for event in member_messages):
@@ -848,8 +747,7 @@ def plan_next_task(
def reconstruct_task_plan(
room_value: Any, events: Sequence[Mapping[str, Any]], task: Mapping[str, Any],
*, local_profiles: Iterable[str],
) -> DiscussionTaskPlan:
*, local_profiles: Iterable[str]) -> DiscussionTaskPlan:
"""Reconstruct and verify one persisted driver task after a restart."""
room = validate_room(room_value, local_profiles=local_profiles)
validated = _validated_events(events, room=room)
@@ -858,9 +756,7 @@ def reconstruct_task_plan(
if not isinstance(identity, driver.TaskIdentity) or not isinstance(payload, Mapping):
raise DiscussionReconstructionError("driver task has no valid identity or payload")
required_payload = frozenset({"target_profile", "prompt", "source_event_seq"})
if not required_payload <= frozenset(payload) or (
frozenset(payload) - required_payload - {"target_member_id"}
):
if not required_payload <= frozenset(payload) or (frozenset(payload) - required_payload - {"target_member_id"}):
raise DiscussionReconstructionError("driver task payload shape changed")
match = _TURN_ID_RE.fullmatch(identity.turn_id)
if match is None:
@@ -868,9 +764,7 @@ def reconstruct_task_plan(
source_event_seq = int(match.group("source"))
if payload.get("source_event_seq") != source_event_seq:
raise DiscussionReconstructionError("task source event does not match turn_id")
discussion = next(
(e for e in validated if e.seq == source_event_seq and e.kind == "message.user"), None
)
discussion = next((e for e in validated if e.seq == source_event_seq and e.kind == "message.user"), None)
if discussion is None:
raise DiscussionReconstructionError("task source user event is missing")
if identity.room_id != room.room_id or identity.thread_id != discussion.payload["thread_id"]:
@@ -880,16 +774,9 @@ def reconstruct_task_plan(
member = next(
(
candidate for candidate in room.members
if (
candidate.member_id == target_member_id
if target_member_id is not None
else candidate.profile == profile
)
),
None,
)
if member is not None and member.profile != profile:
member = None
if candidate.profile == profile
and (target_member_id is None or candidate.member_id == target_member_id)),
None)
if member is None or _member_digest(member) != match.group("member"):
raise DiscussionReconstructionError("task target member does not match turn_id")
prompt = payload.get("prompt")
@@ -898,15 +785,22 @@ def reconstruct_task_plan(
if len(prompt.encode("utf-8")) > driver.MAX_PROMPT_BYTES:
raise DiscussionReconstructionError("task prompt exceeds the driver limit")
reconstructed = _make_task_plan(
room=room, discussion_event=discussion, member=member,
member_index=int(match.group("position")), round_index=int(match.group("round")),
seen_through_seq=int(match.group("seen")), prompt=prompt,
)
room=room, discussion_event=discussion, member=member, member_index=int(match.group("position")),
round_index=int(match.group("round")), seen_through_seq=int(match.group("seen")), prompt=prompt)
if reconstructed.identity != identity or dict(reconstructed.payload) != dict(payload):
raise DiscussionReconstructionError("driver task failed deterministic reconstruction")
return reconstructed
def _turn_coordinates(task: DiscussionTaskPlan) -> dict[str, Any]:
"""The coordinate fields every member message and terminal event carries."""
return {
"discussion_event_id": task.discussion_event_id, "member_id": task.member.member_id,
"member_index": task.member_index, "round_index": task.round_index, "task_id": task.identity.task_id,
"thread_id": task.identity.thread_id, "turn_id": task.identity.turn_id,
}
def _terminal_text(result: Any, *, field: str, fallback: str) -> str:
if isinstance(result, Mapping):
value = result.get(field)
@@ -927,8 +821,7 @@ def _settled_effects(
) -> tuple[dict[str, Any], list[EventPlan]]:
text = _truncate_utf8_text(
_terminal_text(result, field="text", fallback=""),
max_bytes=MAX_MEMBER_TEXT_BYTES, suffix=_TRUNCATED_REPLY_NOTICE,
)
max_bytes=MAX_MEMBER_TEXT_BYTES, suffix=_TRUNCATED_REPLY_NOTICE)
passed = is_pass_text(text)
effects: list[EventPlan] = []
if not passed:
@@ -940,18 +833,8 @@ def _settled_effects(
member_actor["display_name"] = task.member.display_name
effects.append(EventPlan(
event_id=message_event_id, kind="message.member", actor=member_actor,
payload={
"discussion_event_id": task.discussion_event_id,
"member_id": task.member.member_id,
"member_index": task.member_index,
"round_index": task.round_index,
"task_id": task.identity.task_id,
"text": text,
"thread_id": task.identity.thread_id,
"turn_id": task.identity.turn_id,
},
authority_gateway_id=room.gateway_id, authority_epoch=room.authority_epoch,
))
payload={**_turn_coordinates(task), "text": text},
authority_gateway_id=room.gateway_id, authority_epoch=room.authority_epoch))
return {"message_event_id": None if passed else message_event_id, "passed": passed}, effects
@@ -961,25 +844,20 @@ def _failed_effects(result: Any, **_: Any) -> tuple[dict[str, Any], list[EventPl
supplied_reason = (
str(result.get("reason_code") or result.get("reason") or "").strip()
if isinstance(result, Mapping) else ""
)
if isinstance(result, Mapping) else "")
reason_code = supplied_reason if supplied_reason in ALL_REASONS else classify_agent_error(error_text)
return {"error": error_text, "reason_code": reason_code}, []
def _cancelled_effects(
result: Any, *, newer_same_thread: bool, **_: Any
) -> tuple[dict[str, Any], list[EventPlan]]:
def _cancelled_effects(result: Any, *, newer_same_thread: bool, ** _: Any) -> tuple[dict[str, Any], list[EventPlan]]:
reason = (
"superseded_by_newer_user_event" if newer_same_thread
else _terminal_text(result, field="reason", fallback="member turn cancelled")
)
else _terminal_text(result, field="reason", fallback="member turn cancelled"))
return {"reason": reason}, []
def _deferred_effects(
result: Any, *, execution_generation: int | None, **_: Any
) -> tuple[dict[str, Any], list[EventPlan]]:
result: Any, *, execution_generation: int | None, **_: Any) -> tuple[dict[str, Any], list[EventPlan]]:
return {
"execution_generation": execution_generation,
"reason": _terminal_text(result, field="reason", fallback="member_unavailable"),
@@ -987,23 +865,14 @@ def _deferred_effects(
_TERMINAL_EFFECTS = {
"settled": _settled_effects,
"failed": _failed_effects,
"cancelled": _cancelled_effects,
"settled": _settled_effects, "failed": _failed_effects, "cancelled": _cancelled_effects,
"deferred": _deferred_effects,
}
def plan_publication(
room_value: Any,
events: Sequence[Mapping[str, Any]],
task: DiscussionTaskPlan,
*,
status: TerminalKind,
result: Any = None,
execution_generation: int | None = None,
local_profiles: Iterable[str],
) -> PublicationPlan:
room_value: Any, events: Sequence[Mapping[str, Any]], task: DiscussionTaskPlan, *, status: TerminalKind,
result: Any = None, execution_generation: int | None = None, local_profiles: Iterable[str]) -> PublicationPlan:
"""Plan idempotent room effects for one terminal driver task.
A newer user event in the same thread supersedes a late result: the task
@@ -1020,43 +889,26 @@ def plan_publication(
raise DiscussionValidationError("invalid terminal publication status")
if status == "deferred":
message = "deferred publication requires an execution generation"
common.positive_int(execution_generation, error=DiscussionValidationError, message=message)
_bounded_int(execution_generation, message=message, low=1)
newer_same_thread = any(
event.kind == "message.user"
and event.seq > task.seen_through_seq
and event.payload.get("thread_id") == task.identity.thread_id
for event in validated
)
effective_status: TerminalKind = (
"cancelled" if newer_same_thread and status != "deferred" else status
)
for event in validated)
effective_status: TerminalKind = ("cancelled" if newer_same_thread and status != "deferred" else status)
digest = task.identity.task_id.removeprefix("dtask:")
terminal_event_id = (
f"ddeferred:{digest}:g{execution_generation}"
if effective_status == "deferred" else f"dterminal:{digest}"
)
common_payload = {
"discussion_event_id": task.discussion_event_id,
"member_id": task.member.member_id,
"member_index": task.member_index,
"round_index": task.round_index,
"seen_through_seq": task.seen_through_seq,
"task_id": task.identity.task_id,
"thread_id": task.identity.thread_id,
"turn_id": task.identity.turn_id,
}
if effective_status == "deferred" else f"dterminal:{digest}")
common_payload = {**_turn_coordinates(task), "seen_through_seq": task.seen_through_seq}
extra, effects = _TERMINAL_EFFECTS[effective_status](
result, task=task, room=room, message_event_id=f"dmessage:{digest}",
newer_same_thread=newer_same_thread, execution_generation=execution_generation,
)
newer_same_thread=newer_same_thread, execution_generation=execution_generation)
terminal_kind = f"turn.{effective_status}"
effects.append(EventPlan(
event_id=terminal_event_id, kind=terminal_kind,
actor={"kind": "gateway", "id": room.gateway_id},
payload={**common_payload, **extra},
authority_gateway_id=room.gateway_id, authority_epoch=room.authority_epoch,
))
return PublicationPlan(
task_id=task.identity.task_id, terminal_kind=terminal_kind, events=tuple(effects)
)
authority_gateway_id=room.gateway_id, authority_epoch=room.authority_epoch))
return PublicationPlan(task_id=task.identity.task_id, terminal_kind=terminal_kind, events=tuple(effects))
gateway/hosted_room_driver.py
+115 -220
View File
@@ -14,12 +14,12 @@ import json
import math
import sqlite3
from dataclasses import dataclass
from functools import partial
from pathlib import Path
from typing import Any, Callable, Literal, get_args
from gateway.hosted_rooms_common import (
canonical_json, identifier, non_negative_int, positive_int, table_columns, transaction,
)
bounded_int, canonical_json, compact_json, connect, identifier, table_columns, text, transaction)
Clock = Callable[[], float]
TaskStatus = Literal["queued", "running", "settled", "failed", "cancelled", "indeterminate", "deferred", "stopping"]
@@ -38,16 +38,12 @@ _TASK_PAYLOAD_REQUIRED_FIELDS = frozenset({"target_profile", "prompt", "source_e
_TASK_PAYLOAD_OPTIONAL_FIELDS = frozenset({"target_member_id"})
_LEASE_COLUMNS = frozenset({
"room_id", "gateway_id", "authority_epoch", "process_generation",
"lease_generation", "expires_at", "acquired_at", "updated_at", "released_at",
})
"lease_generation", "expires_at", "acquired_at", "updated_at", "released_at"})
_TASK_COLUMN_ORDER = (
"room_id", "task_id", "thread_id", "turn_id", "source_event_seq",
"payload_json", "payload_digest", "status", "execution_generation",
"cancel_generation", "run_gateway_id", "run_process_generation",
"run_lease_generation", "cancel_id", "settlement_id", "settlement_status",
"result_json", "created_at", "updated_at", "started_at", "terminal_at",
"indeterminate_at",
)
"room_id", "task_id", "thread_id", "turn_id", "source_event_seq", "payload_json", "payload_digest",
"status", "execution_generation", "cancel_generation", "run_gateway_id", "run_process_generation",
"run_lease_generation", "cancel_id", "settlement_id", "settlement_status", "result_json", "created_at",
"updated_at", "started_at", "terminal_at", "indeterminate_at")
_TASK_COLUMNS = frozenset(_TASK_COLUMN_ORDER)
_TASK_ORDER = "ORDER BY source_event_seq, created_at, task_id"
_SELECT_LEASE = "SELECT * FROM hosted_room_driver_leases WHERE room_id=?"
@@ -78,79 +74,64 @@ _SETTLE_RUNNING_SQL = _generation_update(_SETTLE_SET, "running") + f" AND {_RUN_
_SETTLE_STOPPING_SQL = _generation_update(_SETTLE_SET, "stopping")
_REQUEUE_RUNNING_SQL = _task_update(
f"{_REQUEUE_SET}, started_at=NULL, updated_at=?",
f"status='running' AND {_GENERATION_FENCE} AND {_RUN_FENCE}",
)
f"status='running' AND {_GENERATION_FENCE} AND {_RUN_FENCE}")
_CANCEL_QUEUED_SQL = _task_update(_CANCEL_SET, "status IN ('queued', 'deferred') AND cancel_generation=?")
_BEGIN_STOP_SQL = _task_update(
"status='stopping', cancel_generation=?, cancel_id=?, updated_at=?",
"status IN ('running', 'indeterminate') AND cancel_generation=?",
)
"status IN ('running', 'indeterminate') AND cancel_generation=?")
_COMPLETE_STOP_SQL = _task_update(
"status='cancelled', terminal_at=?, updated_at=?",
"status='stopping' AND cancel_id=? AND cancel_generation=?",
)
"status='stopping' AND cancel_id=? AND cancel_generation=?")
# Lease-first recovery transitions: name -> (fenced status, SET clause, generation-guard stale message,
# row stale message); the UPDATE is _generation_update(set_clause, status).
_INDETERMINATE_STALE = "indeterminate task generation changed"
_GENERATION_TRANSITIONS = {
"resolve": (
"indeterminate", _SETTLE_SET, _INDETERMINATE_STALE, "indeterminate task changed during reconciliation",
),
"resolve": ("indeterminate", _SETTLE_SET, _INDETERMINATE_STALE, "indeterminate task changed during reconciliation"),
"resolve_cancel": (
"indeterminate", _CANCEL_SET, "indeterminate cancellation proof is stale",
"indeterminate cancellation proof lost its fence",
),
"indeterminate cancellation proof lost its fence"),
"requeue": (
"indeterminate", f"{_REQUEUE_SET}, started_at=NULL, indeterminate_at=NULL, updated_at=?",
_INDETERMINATE_STALE, "indeterminate task changed during requeue",
),
_INDETERMINATE_STALE, "indeterminate task changed during requeue"),
"defer": (
"indeterminate", "status='deferred', result_json=?, terminal_at=?, updated_at=?",
_INDETERMINATE_STALE, "indeterminate task changed during deferral",
),
_INDETERMINATE_STALE, "indeterminate task changed during deferral"),
"requeue_deferred": (
"deferred",
f"{_REQUEUE_SET}, result_json=NULL, started_at=NULL, terminal_at=NULL, indeterminate_at=NULL, updated_at=?",
"deferred task generation changed", "deferred task changed during requeue",
),
}
"deferred task generation changed", "deferred task changed during requeue")}
class DriverStateError(ValueError):
"""Base class for invalid or conflicting driver-state operations."""
class DriverStateError(ValueError): """Base class for invalid or conflicting driver-state operations."""
class DriverValidationError(DriverStateError):
"""Raised when an identifier, clock, TTL, or payload is invalid."""
class DriverValidationError(DriverStateError): """Raised when an identifier, clock, TTL, or payload is invalid."""
class RoomUnavailableError(DriverStateError):
"""Raised when the hosted room does not exist or was disbanded."""
class RoomUnavailableError(DriverStateError): """Raised when the hosted room does not exist or was disbanded."""
class LeaseHeldError(DriverStateError):
"""Raised when another unexpired driver generation owns the room."""
class LeaseHeldError(DriverStateError): """Raised when another unexpired driver generation owns the room."""
class StaleLeaseError(DriverStateError):
"""Raised when a lease generation can no longer mutate room state."""
class StaleLeaseError(DriverStateError): """Raised when a lease generation can no longer mutate room state."""
class TaskConflictError(DriverStateError):
"""Raised when an idempotency key is reused for different task state."""
class TaskConflictError(DriverStateError): """Raised when an idempotency key is reused for different task state."""
class StaleTaskError(DriverStateError):
"""Raised when an obsolete task attempt or cancellation tries to commit."""
class StaleTaskError(DriverStateError): """Raised when an obsolete task attempt or cancellation tries to commit."""
class InvalidTaskTransitionError(DriverStateError):
"""Raised when a requested task transition is not allowed."""
class InvalidTaskTransitionError(DriverStateError): """Raised when a requested task transition is not allowed."""
def _identifier(value: Any, *, label: str) -> str:
return identifier(value, label=label, error=DriverValidationError, max_chars=MAX_IDENTIFIER_CHARS)
_identifier = partial(identifier, error=DriverValidationError, max_chars=MAX_IDENTIFIER_CHARS)
_bounded_int = partial(bounded_int, error=DriverValidationError)
_canonical_json = partial(
canonical_json, error=DriverValidationError, label="result", max_bytes=MAX_RESULT_JSON_BYTES,
ensure_ascii=True)
def _finite(compute: Callable[[], Any], message: str, *, positive: bool = False) -> float:
@@ -180,12 +161,6 @@ def _expiry(now: float, ttl: float) -> float:
return expires_at
def _canonical_json(value: Any) -> str:
return canonical_json(
value, error=DriverValidationError, label="result", max_bytes=MAX_RESULT_JSON_BYTES, ensure_ascii=True
)
def _task_payload(value: Any) -> tuple[dict[str, Any], str, str]:
if not isinstance(value, dict):
raise DriverValidationError("payload must be an object")
@@ -196,21 +171,13 @@ def _task_payload(value: Any) -> tuple[dict[str, Any], str, str]:
if missing:
raise DriverValidationError(f"missing payload fields: {', '.join(sorted(missing))}")
target_profile = _identifier(value["target_profile"], label="target_profile")
prompt = value["prompt"]
if not isinstance(prompt, str):
raise DriverValidationError("prompt must be a string")
if not prompt.strip():
raise DriverValidationError("prompt must not be empty")
if len(prompt.encode("utf-8")) > MAX_PROMPT_BYTES:
raise DriverValidationError("prompt is too large")
source_event_seq = positive_int(
value["source_event_seq"], error=DriverValidationError,
message="source_event_seq must be a positive integer",
)
prompt = text(value["prompt"], error=DriverValidationError, label="prompt", max_bytes=MAX_PROMPT_BYTES, strip=False)
source_event_seq = _bounded_int(
value["source_event_seq"], message="source_event_seq must be a positive integer", low=1)
normalized = {"target_profile": target_profile, "prompt": prompt, "source_event_seq": source_event_seq}
if "target_member_id" in value:
normalized["target_member_id"] = _identifier(value["target_member_id"], label="target_member_id")
encoded = json.dumps(normalized, ensure_ascii=True, sort_keys=True, separators=(",", ":"))
encoded = compact_json(normalized)
return normalized, encoded, hashlib.sha256(encoded.encode("utf-8")).hexdigest()
@@ -267,8 +234,7 @@ def _create_task_table(conn: sqlite3.Connection, table: str = "hosted_room_drive
settlement_id TEXT, settlement_status TEXT, result_json TEXT, created_at REAL NOT NULL,
updated_at REAL NOT NULL, started_at REAL, terminal_at REAL, indeterminate_at REAL,
PRIMARY KEY (room_id, task_id), UNIQUE (room_id, thread_id, turn_id),
FOREIGN KEY (room_id) REFERENCES hosted_rooms(room_id))"""
)
FOREIGN KEY (room_id) REFERENCES hosted_rooms(room_id))""")
def _initialize_schema(conn: sqlite3.Connection) -> None:
@@ -291,8 +257,7 @@ def _validate_schema(conn: sqlite3.Connection) -> None:
if lease_columns != _LEASE_COLUMNS or task_columns != _TASK_COLUMNS:
raise DriverStateError(
"unsupported unpublished hosted-room driver schema; "
"recreate the driver tables before starting the driver"
)
"recreate the driver tables before starting the driver")
for table in ("hosted_room_driver_leases", "hosted_room_driver_tasks"):
foreign_keys = conn.execute(f"PRAGMA foreign_key_list({table})").fetchall()
if not any(row[2] == "hosted_rooms" and row[3] == "room_id" and row[4] == "room_id" for row in foreign_keys):
@@ -307,15 +272,13 @@ def _schema_objects_exist(conn: sqlite3.Connection) -> bool:
if {row[0] for row in rows} != {"hosted_room_driver_leases", "hosted_room_driver_tasks"}:
return False
index = conn.execute(
"SELECT 1 FROM sqlite_master WHERE type='index' AND name='idx_hosted_room_driver_tasks_status'"
).fetchone()
"SELECT 1 FROM sqlite_master WHERE type='index' AND name='idx_hosted_room_driver_tasks_status'").fetchone()
return index is not None
def _task_schema_supports_current_statuses(conn: sqlite3.Connection) -> bool:
row = conn.execute(
"SELECT sql FROM sqlite_master WHERE type='table' AND name='hosted_room_driver_tasks'"
).fetchone()
"SELECT sql FROM sqlite_master WHERE type='table' AND name='hosted_room_driver_tasks'").fetchone()
sql = str(row[0] or "").lower() if row else ""
return "'stopping'" in sql and "'deferred'" in sql
@@ -326,40 +289,33 @@ def _migrate_task_status_constraint(conn: sqlite3.Connection) -> None:
_create_task_table(conn, "hosted_room_driver_tasks_next")
columns = ", ".join(_TASK_COLUMN_ORDER)
conn.execute(
f"INSERT INTO hosted_room_driver_tasks_next ({columns}) SELECT {columns} FROM hosted_room_driver_tasks"
)
f"INSERT INTO hosted_room_driver_tasks_next ({columns}) SELECT {columns} FROM hosted_room_driver_tasks")
conn.execute("DROP TABLE hosted_room_driver_tasks")
conn.execute("ALTER TABLE hosted_room_driver_tasks_next RENAME TO hosted_room_driver_tasks")
conn.execute(_TASK_INDEX_SQL.format(if_not_exists=""))
def _connect(db_path: Path | str) -> sqlite3.Connection:
from hermes_state import apply_wal_with_fallback
"""Open the store; existing tables are validated (after the status-constraint migration when needed).
path = Path(db_path)
path.parent.mkdir(parents=True, exist_ok=True)
conn = sqlite3.connect(path, timeout=10)
conn.row_factory = sqlite3.Row
try:
apply_wal_with_fallback(conn, db_label="state.db (hosted_room_driver)")
conn.execute("PRAGMA foreign_keys=ON")
if _schema_objects_exist(conn):
if not _task_schema_supports_current_statuses(conn):
conn.execute("BEGIN IMMEDIATE")
_migrate_task_status_constraint(conn)
conn.commit()
The driver schema never shipped, so an incompatible draft fails closed in ``_validate_schema``.
"""
existing: list[bool] = []
def ready(conn: sqlite3.Connection) -> bool:
existing.append(_schema_objects_exist(conn))
return existing[0] and _task_schema_supports_current_statuses(conn)
def initialize(conn: sqlite3.Connection) -> None:
(_migrate_task_status_constraint if existing[0] else _initialize_schema)(conn)
conn = connect(db_path, db_label="state.db (hosted_room_driver)", ready=ready, initialize=initialize)
if existing[0]:
try:
_validate_schema(conn)
return conn
# Schema creation is one database-wide transaction. The driver schema
# has never shipped, so an incompatible draft schema fails closed
# instead of attempting a partial in-place migration.
conn.execute("BEGIN IMMEDIATE")
_initialize_schema(conn)
conn.commit()
except Exception:
conn.rollback()
conn.close()
raise
except Exception:
conn.close()
raise
return conn
@@ -369,17 +325,14 @@ def _transaction(db_path: Path | str):
def _lease_from_row(row: sqlite3.Row | dict[str, Any], *, reclaimed: bool = False) -> DriverLease:
return DriverLease(
room_id=row["room_id"], gateway_id=row["gateway_id"],
authority_epoch=int(row["authority_epoch"]), process_generation=row["process_generation"],
lease_generation=int(row["lease_generation"]), expires_at=float(row["expires_at"]),
reclaimed=reclaimed,
)
room_id=row["room_id"], gateway_id=row["gateway_id"], authority_epoch=int(row["authority_epoch"]),
process_generation=row["process_generation"], lease_generation=int(row["lease_generation"]),
expires_at=float(row["expires_at"]), reclaimed=reclaimed)
def _task_identity_from_row(row: sqlite3.Row) -> TaskIdentity:
return TaskIdentity(
room_id=row["room_id"], task_id=row["task_id"], thread_id=row["thread_id"], turn_id=row["turn_id"]
)
room_id=row["room_id"], task_id=row["task_id"], thread_id=row["thread_id"], turn_id=row["turn_id"])
def _optional(cast: Callable[[Any], Any]) -> Callable[[Any], Any]:
@@ -401,8 +354,7 @@ def _task_from_row(row: sqlite3.Row, *, idempotent: bool = False) -> dict[str, A
except (TypeError, json.JSONDecodeError, DriverValidationError) as exc:
raise TaskConflictError("stored task payload is invalid") from exc
if (encoded_payload, payload_digest, payload["source_event_seq"]) != (
row["payload_json"], row["payload_digest"], int(row["source_event_seq"])
):
row["payload_json"], row["payload_digest"], int(row["source_event_seq"])):
raise TaskConflictError("stored task payload failed its integrity check")
task: dict[str, Any] = {"identity": _task_identity_from_row(row), "payload": payload}
for column in _TASK_COLUMN_ORDER[_TASK_COLUMN_ORDER.index("payload_digest"):]:
@@ -430,8 +382,7 @@ def _load_active_room(conn: sqlite3.Connection, room_id: str) -> sqlite3.Row:
try:
row = conn.execute(
"SELECT room_id, authority_gateway_id, authority_epoch, disbanded_at FROM hosted_rooms WHERE room_id=?",
(room_id,),
).fetchone()
(room_id,)).fetchone()
except sqlite3.OperationalError as exc:
if "no such table" in str(exc).lower():
raise RoomUnavailableError("hosted room does not exist") from exc
@@ -444,8 +395,7 @@ def _load_active_room(conn: sqlite3.Connection, room_id: str) -> sqlite3.Row:
def _require_room_authority(
conn: sqlite3.Connection, *, room_id: str, gateway_id: str, authority_epoch: int
) -> sqlite3.Row:
conn: sqlite3.Connection, *, room_id: str, gateway_id: str, authority_epoch: int) -> sqlite3.Row:
room = _load_active_room(conn, room_id)
if room["authority_gateway_id"] != gateway_id or int(room["authority_epoch"]) != authority_epoch:
raise StaleLeaseError("hosted room authority changed")
@@ -454,8 +404,7 @@ def _require_room_authority(
def _require_lease_authority(conn: sqlite3.Connection, lease: DriverLease) -> sqlite3.Row:
return _require_room_authority(
conn, room_id=lease.room_id, gateway_id=lease.gateway_id, authority_epoch=lease.authority_epoch
)
conn, room_id=lease.room_id, gateway_id=lease.gateway_id, authority_epoch=lease.authority_epoch)
def _lease_row_matches(row: sqlite3.Row | None, lease: DriverLease) -> bool:
@@ -485,8 +434,7 @@ def _cancel_generation(value: int) -> int:
def _expected_generations(
lease: DriverLease, identity: TaskIdentity, execution_generation: int, cancel_generation: int
) -> None:
lease: DriverLease, identity: TaskIdentity, execution_generation: int, cancel_generation: int) -> None:
_check_same_room(lease, identity)
if not isinstance(execution_generation, int) or execution_generation < 1:
raise DriverValidationError("expected_execution_generation must be a positive integer")
@@ -515,14 +463,12 @@ def _settlement_replay(settlement_id: str, status: str, result_json: str) -> Cal
def _cancel_replay(cancel_id: str, status: str = "cancelled") -> Callable[[sqlite3.Row], Any]:
"""Replay predicate: same cancel_id already committed in ``status``."""
return lambda row: (
_task_from_row(row, idempotent=True) if row["status"] == status and row["cancel_id"] == cancel_id else None
)
_task_from_row(row, idempotent=True) if row["status"] == status and row["cancel_id"] == cancel_id else None)
def _generations_match(row: sqlite3.Row, status: str, execution_generation: int, cancel_generation: int) -> bool:
return (row["status"], int(row["execution_generation"]), int(row["cancel_generation"])) == (
status, execution_generation, cancel_generation
)
status, execution_generation, cancel_generation)
def _require_cancel_generation(row: sqlite3.Row, expected_cancel_generation: int) -> None:
@@ -534,14 +480,12 @@ def _transition(
db_path: Path | str, identity: TaskIdentity, *, sql: str, set_params: tuple[Any, ...],
fence_params: tuple[Any, ...], stale: str, now: float, lease: DriverLease | None = None,
lease_first: bool = True, replay: Callable[[sqlite3.Row], dict[str, Any] | None] | None = None,
guard: Callable[[sqlite3.Row], None] | None = None,
) -> dict[str, Any]:
guard: Callable[[sqlite3.Row], None] | None = None) -> dict[str, Any]:
"""Run one fenced task transition: load -> idempotent replay -> lease/fence guard -> UPDATE.
``sql`` binds ``(*set_params, room_id, task_id, *fence_params)`` and must hit exactly one row
or ``stale`` is raised. ``lease_first`` puts the active-lease check before the row load
(recovery paths) instead of after the replay (settlement paths: an identical replay still
succeeds after the lease moved on).
``sql`` binds ``(*set_params, room_id, task_id, *fence_params)`` and must hit exactly one row or
``stale`` is raised. ``lease_first`` checks the lease before the row load (recovery paths) instead
of after the replay (settlement paths: an identical replay still succeeds after the lease moved on).
"""
params = (*set_params, identity.room_id, identity.task_id, *fence_params)
with _transaction(db_path) as conn:
@@ -564,8 +508,7 @@ def _transition(
def _generation_transition(
db_path: Path | str, identity: TaskIdentity, lease: DriverLease, name: str, execution_generation: int,
cancel_generation: int, *, now: float, set_params: tuple[Any, ...],
replay: Callable[[sqlite3.Row], Any] | None = None,
) -> dict[str, Any]:
replay: Callable[[sqlite3.Row], Any] | None = None) -> dict[str, Any]:
"""Lease-first transition from ``_GENERATION_TRANSITIONS`` fenced on status + both generations."""
status, set_clause, generation_stale, stale = _GENERATION_TRANSITIONS[name]
@@ -576,18 +519,15 @@ def _generation_transition(
return _transition(
db_path, identity, lease=lease, now=now, replay=replay, guard=guard,
sql=_generation_update(set_clause, status), set_params=set_params,
fence_params=(execution_generation, cancel_generation), stale=stale,
)
fence_params=(execution_generation, cancel_generation), stale=stale)
def _run_fence_transition(
db_path: Path | str, attempt: TaskAttempt, *, guard_stale: str,
lease_generation: Callable[[Any], int] = int, **transition: Any,
) -> dict[str, Any]:
lease_generation: Callable[[Any], int] = int, **transition: Any) -> dict[str, Any]:
"""Transition fenced on this attempt's running generation under its exact lease (row guard + SQL fence).
``lease_generation`` casts the stored run_lease_generation: ``int`` raises on NULL,
``int(v or 0)`` reads it as generation 0.
``lease_generation`` casts the stored run_lease_generation: ``int`` raises on NULL, ``int(v or 0)`` reads 0.
"""
lease = attempt.lease
@@ -596,30 +536,24 @@ def _run_fence_transition(
_generations_match(row, "running", attempt.execution_generation, attempt.cancel_generation)
and row["run_gateway_id"] == lease.gateway_id
and row["run_process_generation"] == lease.process_generation
and lease_generation(row["run_lease_generation"]) == lease.lease_generation
):
and lease_generation(row["run_lease_generation"]) == lease.lease_generation):
raise StaleTaskError(guard_stale)
return _transition(
db_path, attempt.identity, lease=lease, guard=guard,
fence_params=(
attempt.execution_generation, attempt.cancel_generation,
lease.gateway_id, lease.process_generation, lease.lease_generation,
),
**transition,
)
lease.gateway_id, lease.process_generation, lease.lease_generation),
**transition)
def acquire_lease(
db_path: Path | str, *, room_id: Any, gateway_id: Any, authority_epoch: Any, process_generation: Any,
ttl_seconds: Any, clock: Clock,
) -> DriverLease:
ttl_seconds: Any, clock: Clock) -> DriverLease:
"""Acquire an empty or expired room lease with a monotonic generation."""
room_id = _identifier(room_id, label="room_id")
gateway_id = _identifier(gateway_id, label="gateway_id")
authority_epoch = positive_int(
authority_epoch, error=DriverValidationError, message="authority_epoch must be a positive integer"
)
authority_epoch = _bounded_int(authority_epoch, message="authority_epoch must be a positive integer", low=1)
process_generation = _identifier(process_generation, label="process_generation")
ttl_seconds = _ttl(ttl_seconds)
now = _timestamp(clock)
@@ -634,8 +568,7 @@ def acquire_lease(
room_id, gateway_id, authority_epoch, process_generation, lease_generation,
expires_at, acquired_at, updated_at, released_at
) VALUES (?, ?, ?, ?, 1, ?, ?, ?, NULL)""",
(room_id, gateway_id, authority_epoch, process_generation, expires_at, now, now),
)
(room_id, gateway_id, authority_epoch, process_generation, expires_at, now, now))
return _lease_from_row(conn.execute(_SELECT_LEASE, (room_id,)).fetchone())
same_authority = row["gateway_id"] == gateway_id and int(row["authority_epoch"]) == authority_epoch
@@ -644,8 +577,7 @@ def acquire_lease(
renewed_expiry = max(float(row["expires_at"]), expires_at)
conn.execute(
"UPDATE hosted_room_driver_leases SET expires_at=?, updated_at=? WHERE room_id=? AND lease_generation=?",
(renewed_expiry, now, room_id, int(row["lease_generation"])),
)
(renewed_expiry, now, room_id, int(row["lease_generation"])))
return _lease_from_row({**dict(row), "expires_at": renewed_expiry})
if same_authority and live:
raise LeaseHeldError("room driver lease is held by another generation")
@@ -658,9 +590,7 @@ def acquire_lease(
gateway_id != ? OR authority_epoch != ? OR released_at IS NOT NULL OR expires_at <= ?)""",
(
gateway_id, authority_epoch, process_generation, expires_at, now, now,
room_id, int(row["lease_generation"]), gateway_id, authority_epoch, now,
),
)
room_id, int(row["lease_generation"]), gateway_id, authority_epoch, now))
if updated.rowcount != 1:
raise LeaseHeldError("room driver lease changed during acquisition")
return _lease_from_row(conn.execute(_SELECT_LEASE, (room_id,)).fetchone(), reclaimed=True)
@@ -678,8 +608,7 @@ def renew_lease(db_path: Path | str, lease: DriverLease, *, ttl_seconds: Any, cl
"""UPDATE hosted_room_driver_leases SET expires_at=?, updated_at=?
WHERE room_id=? AND gateway_id=? AND process_generation=?
AND lease_generation=? AND released_at IS NULL AND expires_at > ?""",
(expires_at, now, lease.room_id, lease.gateway_id, lease.process_generation, lease.lease_generation, now),
)
(expires_at, now, lease.room_id, lease.gateway_id, lease.process_generation, lease.lease_generation, now))
if updated.rowcount != 1:
raise StaleLeaseError("driver lease changed during renewal")
return dataclasses.replace(lease, expires_at=expires_at, reclaimed=False)
@@ -705,8 +634,7 @@ def release_lease(db_path: Path | str, lease: DriverLease, *, clock: Clock) -> d
conn.execute(
"""UPDATE hosted_room_driver_leases SET expires_at=?, updated_at=?, released_at=?
WHERE room_id=? AND lease_generation=?""",
(now, now, now, lease.room_id, lease.lease_generation),
)
(now, now, now, lease.room_id, lease.lease_generation))
current = {**dict(row), "expires_at": now, "updated_at": now, "released_at": now}
return {"lease": _lease_from_row(current), "idempotent": False}
@@ -726,8 +654,7 @@ def admit_task(db_path: Path | str, identity: TaskIdentity, *, payload: Any, clo
return _task_from_row(existing, idempotent=True)
turn = conn.execute(
"SELECT * FROM hosted_room_driver_tasks WHERE room_id=? AND thread_id=? AND turn_id=?",
(identity.room_id, identity.thread_id, identity.turn_id),
).fetchone()
(identity.room_id, identity.thread_id, identity.turn_id)).fetchone()
if turn is not None:
raise TaskConflictError("thread_id and turn_id are already bound to a task")
conn.execute(
@@ -737,9 +664,7 @@ def admit_task(db_path: Path | str, identity: TaskIdentity, *, payload: Any, clo
) VALUES (?, ?, ?, ?, ?, ?, ?, 'queued', 0, 0, ?, ?)""",
(
identity.room_id, identity.task_id, identity.thread_id, identity.turn_id,
normalized_payload["source_event_seq"], payload_json, payload_digest, now, now,
),
)
normalized_payload["source_event_seq"], payload_json, payload_digest, now, now))
return _task_from_row(_load_task(conn, identity))
@@ -759,14 +684,12 @@ def start_task(
unresolved = conn.execute(
f"""SELECT task_id, status FROM hosted_room_driver_tasks
WHERE room_id=? AND status IN ('running', 'indeterminate', 'stopping') {_TASK_ORDER} LIMIT 1""",
(identity.room_id,),
).fetchone()
(identity.room_id,)).fetchone()
if unresolved is not None:
raise InvalidTaskTransitionError("room recovery must resolve the prior task before starting new work")
next_queued = conn.execute(
f"SELECT task_id FROM hosted_room_driver_tasks WHERE room_id=? AND status='queued' {_TASK_ORDER} LIMIT 1",
(identity.room_id,),
).fetchone()
(identity.room_id,)).fetchone()
if next_queued is None or next_queued["task_id"] != identity.task_id:
raise InvalidTaskTransitionError("task is not next in the hosted room event order")
execution_generation = int(row["execution_generation"]) + 1
@@ -777,15 +700,12 @@ def start_task(
WHERE room_id=? AND task_id=? AND status='queued' AND cancel_generation=?""",
(
execution_generation, lease.gateway_id, lease.process_generation, lease.lease_generation, now, now,
identity.room_id, identity.task_id, expected_cancel_generation,
),
)
identity.room_id, identity.task_id, expected_cancel_generation))
if updated.rowcount != 1:
raise StaleTaskError("task changed during start")
return TaskAttempt(
identity=identity, lease=lease,
execution_generation=execution_generation, cancel_generation=expected_cancel_generation,
)
execution_generation=execution_generation, cancel_generation=expected_cancel_generation)
def settle_task(
@@ -799,8 +719,7 @@ def settle_task(
db_path, attempt, guard_stale="task attempt is stale or cancelled", lease_first=False, now=now,
replay=_settlement_replay(settlement_id, status, result_json),
sql=_SETTLE_RUNNING_SQL, set_params=(status, settlement_id, status, result_json, now, now),
stale="task changed during settlement",
)
stale="task changed during settlement")
def settle_stopping_task(
@@ -818,8 +737,7 @@ def settle_stopping_task(
replay=_settlement_replay(settlement_id, status, result_json),
sql=_SETTLE_STOPPING_SQL, set_params=(status, settlement_id, status, result_json, now, now),
fence_params=(expected_execution_generation, expected_cancel_generation),
stale="task completion lost the stop race",
)
stale="task completion lost the stop race")
def resolve_indeterminate_task(
@@ -834,41 +752,35 @@ def resolve_indeterminate_task(
return _generation_transition(
db_path, identity, lease, "resolve", expected_execution_generation, expected_cancel_generation, now=now,
replay=_settlement_replay(settlement_id, status, result_json),
set_params=(status, settlement_id, status, result_json, now, now),
)
set_params=(status, settlement_id, status, result_json, now, now))
def resolve_indeterminate_cancellation(
db_path: Path | str, identity: TaskIdentity, lease: DriverLease, *, expected_execution_generation: int,
expected_cancel_generation: int, cancel_id: Any, clock: Clock,
) -> dict[str, Any]:
expected_cancel_generation: int, cancel_id: Any, clock: Clock) -> dict[str, Any]:
"""Commit a verified terminal cancellation for an uncertain attempt."""
_expected_generations(lease, identity, expected_execution_generation, expected_cancel_generation)
cancel_id = _identifier(cancel_id, label="cancel_id")
now = _timestamp(clock)
return _generation_transition(
db_path, identity, lease, "resolve_cancel", expected_execution_generation, expected_cancel_generation,
now=now, replay=_cancel_replay(cancel_id), set_params=(expected_cancel_generation + 1, cancel_id, now, now),
)
now=now, replay=_cancel_replay(cancel_id), set_params=(expected_cancel_generation + 1, cancel_id, now, now))
def requeue_indeterminate_task(
db_path: Path | str, identity: TaskIdentity, lease: DriverLease, *, expected_execution_generation: int,
expected_cancel_generation: int, clock: Clock,
) -> dict[str, Any]:
expected_cancel_generation: int, clock: Clock) -> dict[str, Any]:
"""Explicitly retry uncertain work after an operator accepts at-least-once risk."""
_expected_generations(lease, identity, expected_execution_generation, expected_cancel_generation)
now = _timestamp(clock)
return _generation_transition(
db_path, identity, lease, "requeue", expected_execution_generation, expected_cancel_generation, now=now,
set_params=(now,),
)
set_params=(now,))
def defer_indeterminate_task(
db_path: Path | str, identity: TaskIdentity, lease: DriverLease, *, expected_execution_generation: int,
expected_cancel_generation: int, reason: Any, clock: Clock,
) -> dict[str, Any]:
expected_cancel_generation: int, reason: Any, clock: Clock) -> dict[str, Any]:
"""Fence one uncertain attempt and release later room work."""
_expected_generations(lease, identity, expected_execution_generation, expected_cancel_generation)
reason = _identifier(reason, label="defer_reason")
@@ -881,21 +793,18 @@ def defer_indeterminate_task(
return _generation_transition(
db_path, identity, lease, "defer", expected_execution_generation, expected_cancel_generation, now=now,
replay=replay, set_params=(result_json, now, now),
)
replay=replay, set_params=(result_json, now, now))
def requeue_deferred_task(
db_path: Path | str, identity: TaskIdentity, lease: DriverLease, *, expected_execution_generation: int,
expected_cancel_generation: int, clock: Clock,
) -> dict[str, Any]:
expected_cancel_generation: int, clock: Clock) -> dict[str, Any]:
"""Explicitly retry a fenced deferred turn under a new generation."""
_expected_generations(lease, identity, expected_execution_generation, expected_cancel_generation)
now = _timestamp(clock)
return _generation_transition(
db_path, identity, lease, "requeue_deferred", expected_execution_generation, expected_cancel_generation,
now=now, set_params=(now,),
)
now=now, set_params=(now,))
def requeue_not_admitted_task(db_path: Path | str, attempt: TaskAttempt, *, clock: Clock) -> dict[str, Any]:
@@ -908,15 +817,13 @@ def requeue_not_admitted_task(db_path: Path | str, attempt: TaskAttempt, *, cloc
_generations_match(row, "queued", attempt.execution_generation, attempt.cancel_generation)
and row["run_gateway_id"] is None
and row["run_process_generation"] is None
and row["run_lease_generation"] is None
)
and row["run_lease_generation"] is None)
return _task_from_row(row, idempotent=True) if requeued else None
return _run_fence_transition(
db_path, attempt, guard_stale="not-admitted task attempt lost its fence",
lease_generation=lambda value: int(value or 0), now=now, replay=replay,
sql=_REQUEUE_RUNNING_SQL, set_params=(now,), stale="not-admitted task changed during requeue",
)
sql=_REQUEUE_RUNNING_SQL, set_params=(now,), stale="not-admitted task changed during requeue")
def cancel_task(
@@ -937,8 +844,7 @@ def cancel_task(
return _transition(
db_path, identity, now=now, replay=_cancel_replay(cancel_id), guard=guard, sql=_CANCEL_QUEUED_SQL,
set_params=(expected_cancel_generation + 1, cancel_id, now, now), fence_params=(expected_cancel_generation,),
stale="task changed during cancellation",
)
stale="task changed during cancellation")
def begin_task_cancel(
@@ -957,8 +863,7 @@ def begin_task_cancel(
return _transition(
db_path, identity, now=now, replay=_cancel_replay(cancel_id, "stopping"), guard=guard, sql=_BEGIN_STOP_SQL,
set_params=(expected_cancel_generation + 1, cancel_id, now), fence_params=(expected_cancel_generation,),
stale="task changed during stop request",
)
stale="task changed during stop request")
def complete_task_cancel(
@@ -970,15 +875,13 @@ def complete_task_cancel(
def guard(row: sqlite3.Row) -> None:
if (row["status"], row["cancel_id"], int(row["cancel_generation"])) != (
"stopping", cancel_id, expected_cancel_generation
):
"stopping", cancel_id, expected_cancel_generation):
raise StaleTaskError("task stop acknowledgement is stale")
return _transition(
db_path, identity, now=now, replay=_cancel_replay(cancel_id), guard=guard, sql=_COMPLETE_STOP_SQL,
set_params=(now, now), fence_params=(cancel_id, expected_cancel_generation),
stale="task changed during stop acknowledgement",
)
stale="task changed during stop acknowledgement")
def recover_room(db_path: Path | str, lease: DriverLease, *, clock: Clock) -> dict[str, list[TaskIdentity]]:
@@ -989,18 +892,15 @@ def recover_room(db_path: Path | str, lease: DriverLease, *, clock: Clock) -> di
with _transaction(db_path) as conn:
_require_active_lease(conn, lease, now=now)
stale_rows = conn.execute(
f"SELECT * FROM hosted_room_driver_tasks WHERE {foreign_running} {_TASK_ORDER}", fence
).fetchall()
f"SELECT * FROM hosted_room_driver_tasks WHERE {foreign_running} {_TASK_ORDER}", fence).fetchall()
if stale_rows:
conn.execute(
f"""UPDATE hosted_room_driver_tasks SET status='indeterminate', indeterminate_at=?, updated_at=?
WHERE {foreign_running}""",
(now, now, *fence),
)
(now, now, *fence))
return {
status: [_task_identity_from_row(row) for row in _tasks_in_order(conn, lease.room_id, status)]
for status in ("queued", "indeterminate")
}
for status in ("queued", "indeterminate")}
def _read(db_path: Path | str, query: Callable[[sqlite3.Connection], Any]) -> Any:
@@ -1026,19 +926,17 @@ def list_tasks(db_path: Path | str, *, room_id: Any, status: TaskStatus | None =
def prune_published_terminal_tasks(
db_path: Path | str, *, room_id: Any, clock: Clock,
retention_seconds: float = TERMINAL_TASK_RETENTION_SECONDS, retain: int = MAX_RETAINED_TERMINAL_TASKS,
) -> int:
retention_seconds: float = TERMINAL_TASK_RETENTION_SECONDS, retain: int = MAX_RETAINED_TERMINAL_TASKS) -> int:
"""Bound execution rows after outcomes are durable in the room log."""
room_id = _identifier(room_id, label="room_id")
now = _timestamp(clock)
if retention_seconds <= 0:
raise DriverValidationError("retention_seconds must be positive")
non_negative_int(retain, error=DriverValidationError, message="retain must be a non-negative integer")
_bounded_int(retain, message="retain must be a non-negative integer")
with _transaction(db_path) as conn:
publications = conn.execute(
"SELECT 1 FROM sqlite_master WHERE type='table' AND name='hosted_room_policy_publications'"
).fetchone()
"SELECT 1 FROM sqlite_master WHERE type='table' AND name='hosted_room_policy_publications'").fetchone()
if publications is None:
return 0
rows = conn.execute(
@@ -1047,9 +945,7 @@ def prune_published_terminal_tasks(
AND EXISTS (SELECT 1 FROM hosted_room_policy_publications p
WHERE p.room_id=t.room_id AND p.task_id=t.task_id
AND p.kind IN ('turn.settled', 'turn.failed', 'turn.cancelled'))
ORDER BY t.terminal_at DESC, t.task_id ASC""",
(room_id,),
).fetchall()
ORDER BY t.terminal_at DESC, t.task_id ASC""", (room_id,)).fetchall()
cutoff = now - float(retention_seconds)
candidates = [
str(row["task_id"])
@@ -1061,6 +957,5 @@ def prune_published_terminal_tasks(
placeholders = ",".join("?" for _ in candidates)
deleted = conn.execute(
f"DELETE FROM hosted_room_driver_tasks WHERE room_id=? AND task_id IN ({placeholders})",
(room_id, *candidates),
)
(room_id, *candidates))
return max(0, int(deleted.rowcount))
gateway/hosted_room_execution_policy.py
+9 -24
View File
@@ -14,13 +14,10 @@ POLICY_VERSION = 1
MAX_POLICY_TOOLSETS = 128
MAX_POLICY_ITERATIONS = (1 << 53) - 1
_IDENTIFIER_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]*$")
_POLICY_FIELDS = {
"version", "target_profile", "enabled_toolsets", "approval_mode", "max_iterations", "policy_digest",
}
_POLICY_FIELDS = {"version", "target_profile", "enabled_toolsets", "approval_mode", "max_iterations", "policy_digest"}
class RoomExecutionPolicyError(ValueError):
"""A RoomLink execution policy is malformed or no longer current."""
class RoomExecutionPolicyError(ValueError): """A RoomLink execution policy is malformed or no longer current."""
def _policy_digest(unsigned: Mapping[str, Any]) -> str:
@@ -67,15 +64,11 @@ class RoomExecutionPolicy:
if (
isinstance(max_iterations, bool)
or not isinstance(max_iterations, int)
or not 1 <= max_iterations <= MAX_POLICY_ITERATIONS
):
or not 1 <= max_iterations <= MAX_POLICY_ITERATIONS):
raise RoomExecutionPolicyError("max_iterations is invalid")
unsigned = {
"version": POLICY_VERSION,
"target_profile": target_profile,
"enabled_toolsets": list(toolsets),
"approval_mode": approval_mode,
"max_iterations": max_iterations,
"version": POLICY_VERSION, "target_profile": target_profile, "enabled_toolsets": list(toolsets),
"approval_mode": approval_mode, "max_iterations": max_iterations,
}
supplied = str(value["policy_digest"] or "").strip().lower()
if supplied != _policy_digest(unsigned):
@@ -103,12 +96,9 @@ def execution_policy_mapping(*, target_profile: str, config: Mapping[str, Any] |
agent = config.get("agent") if isinstance(config.get("agent"), Mapping) else {}
approvals = config.get("approvals") if isinstance(config.get("approvals"), Mapping) else {}
unsigned = {
"version": POLICY_VERSION,
"target_profile": _identifier(target_profile, field="target_profile"),
"version": POLICY_VERSION, "target_profile": _identifier(target_profile, field="target_profile"),
"enabled_toolsets": toolsets,
"approval_mode": (
"off" if _YOLO_MODE_FROZEN else _normalize_approval_mode(approvals.get("mode", "manual"))
),
"approval_mode": ("off" if _YOLO_MODE_FROZEN else _normalize_approval_mode(approvals.get("mode", "manual"))),
"max_iterations": min(resolve_turn_limit(agent.get("max_turns")), MAX_POLICY_ITERATIONS),
}
value = {**unsigned, "policy_digest": _policy_digest(unsigned)}
@@ -131,12 +121,7 @@ def current_room_execution_policy() -> RoomExecutionPolicy | None:
__all__ = [
"MAX_POLICY_ITERATIONS",
"POLICY_VERSION",
"RoomExecutionPolicy",
"RoomExecutionPolicyError",
"bind_room_execution_policy",
"current_room_execution_policy",
"execution_policy_mapping",
"MAX_POLICY_ITERATIONS", "POLICY_VERSION", "RoomExecutionPolicy", "RoomExecutionPolicyError",
"bind_room_execution_policy", "current_room_execution_policy", "execution_policy_mapping",
"reset_room_execution_policy",
]
gateway/hosted_room_links.py
+20 -34
View File
@@ -12,6 +12,7 @@ import json
import os
import time
from dataclasses import dataclass, field
from functools import partial
from pathlib import Path
from typing import Any, Mapping
@@ -20,23 +21,21 @@ from gateway.hosted_room_peer import (
GatewayRoomCatalog,
HostedRoomPeerError,
TransportSecurity,
validate_room_link_url,
)
validate_room_link_url)
from gateway.hosted_rooms_common import compact_json, exact_fields
MAX_LINKS = 512
MAX_GRANT_CHARS = 16 * 1024
_LEGACY_FIELDS = {
"room_id", "member_id", "target_url", "target_profile", "grant", "catalog",
"cancellation_scope_id", "trace_id", "updated_at",
}
"cancellation_scope_id", "trace_id", "updated_at"}
_OPTIONAL_FIELDS = {"transport_security", "status"}
# SQLite record columns that map 1:1 onto mapping fields, in record order
# (``catalog_json`` is the serialized ``catalog``).
_RECORD_FIELDS = (
"room_id", "member_id", "target_url", "target_profile", "grant", "cancellation_scope_id",
"trace_id", "transport_security", "status", "updated_at",
)
"trace_id", "transport_security", "status", "updated_at")
_STATUSES = {"ready", "unavailable", "needs_reauthorization"}
@@ -56,8 +55,7 @@ class StoredRoomLink:
@classmethod
def from_mapping(cls, value: Mapping[str, Any]) -> "StoredRoomLink":
if set(value) - _LEGACY_FIELDS - _OPTIONAL_FIELDS or not _LEGACY_FIELDS.issubset(value):
raise HostedRoomPeerError("stored room link fields are invalid")
_link_fields(value)
room_id = _short_string(value["room_id"], "room_id")
member_id = _short_string(value["member_id"], "member_id")
target_profile = _short_string(value["target_profile"], "target_profile")
@@ -85,8 +83,7 @@ class StoredRoomLink:
trace_id=_short_string(value["trace_id"], "trace_id"),
transport_security=transport_security, # type: ignore[arg-type]
status=status,
updated_at=updated_at,
)
updated_at=updated_at)
@classmethod
def from_record(cls, value: Mapping[str, Any]) -> "StoredRoomLink":
@@ -101,10 +98,16 @@ class StoredRoomLink:
def as_record(self) -> dict[str, Any]:
record = {name: getattr(self, name) for name in _RECORD_FIELDS}
record["catalog_json"] = json.dumps(self.catalog_mapping(), sort_keys=True, separators=(",", ":"))
record["catalog_json"] = compact_json(self.catalog_mapping(), ensure_ascii=False)
return record
_link_fields = partial(
exact_fields, label="stored room link", required=_LEGACY_FIELDS, optional=_OPTIONAL_FIELDS,
error=HostedRoomPeerError, not_object="stored room link fields are invalid",
missing_fmt="stored room link fields are invalid", unknown_fmt="stored room link fields are invalid")
def _short_string(value: Any, field: str) -> str:
normalized = str(value or "").strip()
if not normalized or len(normalized) > 256:
@@ -149,32 +152,15 @@ def mark_room_link_status(db_path: Path | str, *, room_id: str, member_id: str,
raise HostedRoomPeerError("stored room link status is invalid")
return hosted_rooms.update_room_link_status(
db_path, room_id=_short_string(room_id, "room_id"), member_id=_short_string(member_id, "member_id"),
status=status,
)
status=status)
def make_stored_link(
*,
room_id: str,
member_id: str,
target_url: str,
target_profile: str,
grant: str,
catalog: GatewayRoomCatalog,
cancellation_scope_id: str,
trace_id: str,
) -> StoredRoomLink:
*, room_id: str, member_id: str, target_url: str, target_profile: str, grant: str,
catalog: GatewayRoomCatalog, cancellation_scope_id: str, trace_id: str) -> StoredRoomLink:
target_url, transport_security = validate_room_link_url(target_url)
return StoredRoomLink.from_mapping({
"room_id": room_id,
"member_id": member_id,
"target_url": target_url,
"target_profile": target_profile,
"grant": grant,
"catalog": catalog.as_mapping(),
"cancellation_scope_id": cancellation_scope_id,
"trace_id": trace_id,
"transport_security": transport_security,
"status": "ready",
"updated_at": time.time(),
"room_id": room_id, "member_id": member_id, "target_url": target_url, "target_profile": target_profile,
"grant": grant, "catalog": catalog.as_mapping(), "cancellation_scope_id": cancellation_scope_id,
"trace_id": trace_id, "transport_security": transport_security, "status": "ready", "updated_at": time.time(),
})
gateway/hosted_room_peer.py
+42 -87
View File
@@ -16,7 +16,6 @@ import math
import os
import re
import stat
import time
import urllib.parse
from dataclasses import asdict, dataclass
from functools import lru_cache, partial
@@ -24,7 +23,7 @@ from pathlib import Path
from typing import Any, Callable, Iterable, Literal, Mapping
from gateway.hosted_room_execution_policy import RoomExecutionPolicy, execution_policy_mapping
from gateway.hosted_rooms_common import exact_fields, identifier, positive_int
from gateway.hosted_rooms_common import bounded_int, clock, compact_json, exact_fields, identifier, text
# v2 adds authority/member lineage to scoped grants and is deliberately not
@@ -40,12 +39,10 @@ LinkMode = Literal["direct", "overlay", "relay", "pull", "desktop"]
TransportSecurity = Literal["tls", "loopback"]
class HostedRoomPeerError(ValueError):
"""Base error for malformed or unauthorized peer-room input."""
class HostedRoomPeerError(ValueError): """Base error for malformed or unauthorized peer-room input."""
class HostedRoomGrantError(HostedRoomPeerError):
"""Raised when a room-scoped grant is invalid or expired."""
class HostedRoomGrantError(HostedRoomPeerError): """Raised when a room-scoped grant is invalid or expired."""
_ROOM_GRANT_SECRET_FILE = ".room-link-grant-secret"
@@ -104,10 +101,9 @@ def _gateway_room_grant_secret_for_home(home_value: str) -> bytes:
def gateway_room_grant_secret(root: Path | str | None = None) -> bytes:
"""Load or atomically mint the gateway-only RoomLink signing secret.
API keys are bearer credentials known to clients and may be profile scoped;
they must never become grant-signing authority. This secret lives in the
installation root, is not exposed by configuration or capability RPCs, and
is shared only by the gateway processes that serve this installation.
API keys are client-known, possibly profile-scoped bearer credentials and must never become
grant-signing authority; this secret lives in the installation root, is never exposed by
config or capability RPCs, and is shared only by this installation's gateway processes.
"""
if root is None:
from hermes_constants import get_hermes_home
@@ -121,8 +117,7 @@ def gateway_room_grant_secret(root: Path | str | None = None) -> bytes:
def derive_room_grant_secret(api_key: str) -> bytes:
"""Domain-separate room grants from the configured API key.
The API-server startup guard enforces the production key-strength policy;
this helper keeps a lower structural floor for isolated contract tests.
Production key strength is enforced by the API-server startup guard; this floor is for contract tests.
"""
if not isinstance(api_key, str) or len(api_key) < 8:
raise HostedRoomGrantError("room grants require a strong gateway API key")
@@ -132,12 +127,12 @@ def derive_room_grant_secret(api_key: str) -> bytes:
def _identifier(value: Any, *, field: str) -> str:
return identifier(
value, label=field, error=HostedRoomPeerError, max_chars=256, pattern=_IDENTIFIER_RE,
invalid=f"{field} is invalid",
)
invalid=f"{field} is invalid")
def _positive_int(value: Any, *, field: str) -> int:
return positive_int(value, error=HostedRoomPeerError, message=f"{field} must be a positive integer")
return bounded_int(
value, error=HostedRoomPeerError, message=f"{field} must be a positive integer", low=1)
def _digest(value: Any, *, field: str) -> str:
@@ -148,12 +143,11 @@ def _digest(value: Any, *, field: str) -> str:
_exact_fields = partial(
exact_fields, error=HostedRoomPeerError, missing_fmt="{label} missing fields: {fields}",
unknown_fmt="{label} unknown fields: {fields}",
)
unknown_fmt="{label} unknown fields: {fields}")
def _canonical_json(value: Mapping[str, Any]) -> bytes:
return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode("ascii")
return compact_json(value).encode("ascii")
def _b64encode(value: bytes) -> str:
@@ -198,8 +192,7 @@ def _parse_endpoint(endpoint: Any) -> tuple[str | None, str | None, TransportSec
_CATALOG_FIELDS = {
"installation_id", "protocol_versions", "link_modes", "persistent_process", "text",
"attachments", "execution_policy", "catalog_digest",
}
"attachments", "execution_policy", "catalog_digest"}
@dataclass(frozen=True)
@@ -239,9 +232,7 @@ class GatewayRoomCatalog:
persistent_process=value["persistent_process"], text=value["text"],
attachments=value["attachments"], execution_policy=policy,
catalog_digest=_digest(value["catalog_digest"], field="catalog_digest"),
endpoint_url=endpoint_url, endpoint_reason=endpoint_reason,
transport_security=transport_security,
)
endpoint_url=endpoint_url, endpoint_reason=endpoint_reason, transport_security=transport_security)
unsigned = catalog.as_mapping()
del unsigned["catalog_digest"]
expected = hashlib.sha256(_canonical_json(unsigned)).hexdigest()
@@ -255,8 +246,7 @@ class GatewayRoomCatalog:
"installation_id": self.installation_id, "protocol_versions": list(self.protocol_versions),
"link_modes": list(self.link_modes), "persistent_process": self.persistent_process,
"text": self.text, "attachments": self.attachments,
"execution_policy": self.execution_policy.as_mapping(), "catalog_digest": self.catalog_digest,
}
"execution_policy": self.execution_policy.as_mapping(), "catalog_digest": self.catalog_digest}
if self.endpoint_url is not None or self.endpoint_reason is not None:
value["endpoint"] = self.endpoint_mapping()
return value
@@ -264,9 +254,7 @@ class GatewayRoomCatalog:
def endpoint_mapping(self) -> dict[str, Any]:
"""Return the normalized self-advertised endpoint capability."""
if self.endpoint_url is not None:
return {
"available": True, "url": self.endpoint_url, "transport_security": self.transport_security,
}
return {"available": True, "url": self.endpoint_url, "transport_security": self.transport_security}
return {"available": False, "reason": self.endpoint_reason or "not_configured"}
@@ -274,18 +262,14 @@ def catalog_mapping(
*, installation_id: str, protocol_versions: Iterable[int] = (PROTOCOL_VERSION,),
link_modes: Iterable[LinkMode] = ("direct", "pull"), persistent_process: bool, text: bool = True,
attachments: bool = False, endpoint: Mapping[str, Any] | None = None, target_profile: str | None = None,
execution_policy: Mapping[str, Any] | None = None,
) -> dict[str, Any]:
execution_policy: Mapping[str, Any] | None = None) -> dict[str, Any]:
"""Build a canonical catalog mapping with its digest."""
# A Desktop-managed gateway exits with the app: the caller's flag is only an
# upper bound, so call sites that still pass ``True`` stay honest.
persistent_process = bool(persistent_process and os.getenv("HERMES_DESKTOP") != "1")
profile = (
str(target_profile or "").strip() or (os.getenv("HERMES_PROFILE") or "default").strip() or "default"
)
profile = (str(target_profile or "").strip() or (os.getenv("HERMES_PROFILE") or "default").strip() or "default")
checked_policy = RoomExecutionPolicy.from_mapping(
execution_policy or execution_policy_mapping(target_profile=profile)
)
execution_policy or execution_policy_mapping(target_profile=profile))
# A RoomLink run is initiated by another installation. Process-wide YOLO
# mode bypasses the scoped approval ContextVar, so rewriting the advertised
# policy cannot make it safe: refuse until manual or smart approvals are on.
@@ -293,16 +277,13 @@ def catalog_mapping(
raise HostedRoomPeerError("remote room execution requires manual or smart approvals")
value = {
"installation_id": _identifier(installation_id, field="installation_id"),
"protocol_versions": sorted(
{_positive_int(item, field="protocol_version") for item in protocol_versions}
),
"protocol_versions": sorted({_positive_int(item, field="protocol_version") for item in protocol_versions}),
# Direct HTTPS/loopback is the only RoomLink transport implemented by
# this backend slice. Do not advertise pull/relay placeholders.
"link_modes": [mode for mode in dict.fromkeys(link_modes) if mode == "direct"],
"persistent_process": persistent_process, "text": bool(text), "attachments": bool(attachments),
"execution_policy": checked_policy.as_mapping(),
"endpoint": dict(local_room_link_endpoint() if endpoint is None else endpoint),
}
"endpoint": dict(local_room_link_endpoint() if endpoint is None else endpoint)}
value["catalog_digest"] = hashlib.sha256(_canonical_json(value)).hexdigest()
GatewayRoomCatalog.from_mapping(value)
return value
@@ -311,14 +292,12 @@ def catalog_mapping(
def local_catalog_mapping(
*, installation_id: str, protocol_versions: Iterable[int] = (PROTOCOL_VERSION,),
link_modes: Iterable[LinkMode] = ("direct", "pull"), text: bool = True, attachments: bool = False,
target_profile: str | None = None, execution_policy: Mapping[str, Any] | None = None,
) -> dict[str, Any]:
target_profile: str | None = None, execution_policy: Mapping[str, Any] | None = None) -> dict[str, Any]:
"""Build the one truthful catalog advertised by this local process."""
return catalog_mapping(
installation_id=installation_id, protocol_versions=protocol_versions, link_modes=link_modes,
persistent_process=True, text=text, attachments=attachments, target_profile=target_profile,
execution_policy=execution_policy,
)
execution_policy=execution_policy)
def local_room_link_endpoint(value: Any | None = None) -> dict[str, Any]:
@@ -369,11 +348,7 @@ def _configured_room_link_url() -> str | None:
def validate_room_link_url(value: Any) -> tuple[str, TransportSecurity]:
"""Validate a RoomLink endpoint and classify its transport protection.
Plaintext HTTP is allowed only toward the local loopback interface; every
non-loopback endpoint must use HTTPS.
"""
"""Validate a RoomLink endpoint and classify its transport: plaintext HTTP only toward loopback."""
raw = str(value or "").strip().rstrip("/")
try:
parsed = urllib.parse.urlsplit(raw)
@@ -445,27 +420,23 @@ class HostedMemberDispatch:
prompt = value["prompt"]
if not isinstance(prompt, str) or not prompt.strip():
raise HostedRoomPeerError("prompt must be a non-empty string")
if len(prompt.encode("utf-8")) > MAX_PROMPT_BYTES:
raise HostedRoomPeerError("prompt is too large")
text(prompt, error=HostedRoomPeerError, label="prompt", max_bytes=MAX_PROMPT_BYTES, strip=False)
expected_prompt_digest = hashlib.sha256(prompt.encode("utf-8")).hexdigest()
prompt_digest = _digest(value["prompt_digest"], field="prompt_digest")
if not hmac.compare_digest(expected_prompt_digest, prompt_digest):
raise HostedRoomPeerError("prompt_digest does not match prompt")
return cls(
prompt=prompt, prompt_digest=prompt_digest,
**{name: check(value[name], field=name) for name, check in _DISPATCH_FIELDS.items()},
)
**{name: check(value[name], field=name) for name, check in _DISPATCH_FIELDS.items()})
# Grant scope fields, in issue-time validation order; each is validated with
# the same checker as the matching dispatch field (``grant_id`` is an identifier).
_GRANT_SCOPE = (
"grant_id", "room_id", "home_install_id", "authority_gateway_id", "authority_epoch",
"member_id", "target_install_id", "target_profile",
)
"member_id", "target_install_id", "target_profile")
_GRANT_FIELDS = frozenset({
"version", *_GRANT_SCOPE, "execution_policy_digest", "permissions", "issued_at", "expires_at",
})
"version", *_GRANT_SCOPE, "execution_policy_digest", "permissions", "issued_at", "expires_at"})
_GRANT_REFRESH_FIELDS = _GRANT_FIELDS | {"status_expires_at"}
_GRANT_PERMISSIONS = {"approve", "dispatch", "status", "stop"}
MAX_DISPATCH_GRANT_TTL_SECONDS = 24 * 60 * 60
@@ -476,24 +447,20 @@ def issue_room_grant(
secret: bytes, *, grant_id: str, room_id: str, home_install_id: str, authority_gateway_id: str,
authority_epoch: int, member_id: str, target_install_id: str, target_profile: str,
execution_policy_digest: str | None = None,
permissions: Iterable[str] = ("approve", "dispatch", "status", "stop"),
issued_at: float | None = None, ttl_seconds: float = 3600, status_ttl_seconds: float | None = None,
status_expires_at: float | None = None,
) -> str:
permissions: Iterable[str] = ("approve", "dispatch", "status", "stop"), issued_at: float | None = None,
ttl_seconds: float = 3600, status_ttl_seconds: float | None = None, status_expires_at: float | None = None) -> str:
"""Issue a target-verifiable bearer grant scoped to one room member."""
if len(secret) < 32:
raise HostedRoomGrantError("room grant secret must be at least 32 bytes")
now = time.time() if issued_at is None else float(issued_at)
now = clock(issued_at)
bounded_status_expiry = (
now + float(ttl_seconds if status_ttl_seconds is None else status_ttl_seconds)
if status_expires_at is None
else float(status_expires_at)
)
else float(status_expires_at))
if (
not math.isfinite(now) or ttl_seconds <= 0 or ttl_seconds > MAX_DISPATCH_GRANT_TTL_SECONDS
or not math.isfinite(bounded_status_expiry) or bounded_status_expiry < now + float(ttl_seconds)
or bounded_status_expiry > now + MAX_STATUS_GRANT_TTL_SECONDS
):
or bounded_status_expiry > now + MAX_STATUS_GRANT_TTL_SECONDS):
raise HostedRoomGrantError("room grant lifetime is invalid")
allowed = tuple(sorted(set(permissions)))
if not allowed or not set(allowed) <= _GRANT_PERMISSIONS:
@@ -501,19 +468,16 @@ def issue_room_grant(
scope = {
"grant_id": grant_id, "room_id": room_id, "home_install_id": home_install_id,
"authority_gateway_id": authority_gateway_id, "authority_epoch": authority_epoch,
"member_id": member_id, "target_install_id": target_install_id, "target_profile": target_profile,
}
"member_id": member_id, "target_install_id": target_install_id, "target_profile": target_profile}
payload = {
"version": PROTOCOL_VERSION,
**{name: _DISPATCH_FIELDS.get(name, _identifier)(scope[name], field=name) for name in _GRANT_SCOPE},
"execution_policy_digest": _digest(
execution_policy_digest
or execution_policy_mapping(target_profile=target_profile)["policy_digest"],
field="execution_policy_digest",
),
field="execution_policy_digest"),
"permissions": list(allowed), "issued_at": now, "expires_at": now + float(ttl_seconds),
"status_expires_at": bounded_status_expiry,
}
"status_expires_at": bounded_status_expiry}
encoded = _canonical_json(payload)
signature = hmac.new(secret, encoded, hashlib.sha256).digest()
token = f"{_b64encode(encoded)}.{_b64encode(signature)}"
@@ -524,8 +488,7 @@ def issue_room_grant(
def verify_room_grant(
secret: bytes, token: str, dispatch: HostedMemberDispatch, *, permission: str = "dispatch",
now: float | None = None,
) -> dict[str, Any]:
now: float | None = None) -> dict[str, Any]:
"""Verify one room grant against exact recipient dispatch coordinates."""
payload = decode_room_grant(secret, token, permission=permission, now=now)
if payload["version"] != dispatch.protocol_version:
@@ -536,9 +499,7 @@ def verify_room_grant(
return payload
def decode_room_grant(
secret: bytes, token: str, *, permission: str, now: float | None = None
) -> dict[str, Any]:
def decode_room_grant(secret: bytes, token: str, *, permission: str, now: float | None = None) -> dict[str, Any]:
"""Verify grant signature, lifetime and operation without a dispatch."""
if not isinstance(token, str) or len(token.encode("utf-8")) > MAX_TOKEN_BYTES:
raise HostedRoomGrantError("room grant is invalid")
@@ -554,7 +515,7 @@ def decode_room_grant(
raise HostedRoomGrantError("room grant payload is invalid") from exc
if not isinstance(payload, dict) or frozenset(payload) not in {_GRANT_FIELDS, _GRANT_REFRESH_FIELDS}:
raise HostedRoomGrantError("room grant fields are invalid")
checked_now = time.time() if now is None else float(now)
checked_now = clock(now)
if not math.isfinite(checked_now):
raise HostedRoomGrantError("room grant clock is invalid")
try:
@@ -575,18 +536,12 @@ def decode_room_grant(
return payload
def room_grant_needs_dispatch_refresh(
token: str, *, now: float | None = None, leeway_seconds: float = 5 * 60
) -> bool:
"""Read only grant timing to schedule target-validated refresh.
This deliberately does not establish trust; the target validates the
signature and immutable scope before issuing a replacement.
"""
def room_grant_needs_dispatch_refresh(token: str, *, now: float | None = None, leeway_seconds: float = 5 * 60) -> bool:
"""Read only grant timing to schedule refresh; trust is established by the target, not here."""
try:
payload = json.loads(_b64decode(_split_token(token)[0]).decode("ascii"))
expires_at = float(payload["expires_at"])
checked_now = time.time() if now is None else float(now)
checked_now = clock(now)
return checked_now + max(0.0, float(leeway_seconds)) >= expires_at
except Exception:
return True
gateway/hosted_room_policy_checkpoint.py
+51 -129
View File
@@ -14,6 +14,7 @@ from pathlib import Path
from typing import Any, Callable, Mapping
from gateway import hosted_rooms
from gateway.hosted_rooms_common import compact_json
MAX_ACTIVE_POLICY_EVENTS = 64
@@ -52,12 +53,8 @@ _SCHEMA_DDL = (
room_id TEXT PRIMARY KEY, schema_version INTEGER NOT NULL)""",
)
_ROOM_EVENT_COLUMNS = (
"room_id, seq, event_id, kind, actor_json, authority_epoch, payload_json, created_at"
)
_DELETE_ACTIVE_EVENTS_SQL = (
"DELETE FROM hosted_room_policy_events WHERE room_id=? AND discussion_event_id=?"
)
_ROOM_EVENT_COLUMNS = hosted_rooms._EVENT_COLUMNS
_DELETE_ACTIVE_EVENTS_SQL = ("DELETE FROM hosted_room_policy_events WHERE room_id=? AND discussion_event_id=?")
_TRANSCRIPT_EVENTS_SQL = f"""WITH transcript_events(seq) AS (
SELECT seq FROM hosted_room_policy_transcript WHERE room_id=? AND thread_id=?
UNION ALL
@@ -79,13 +76,7 @@ class PolicySnapshot:
watermarks: Mapping[tuple[str, str], int]
def _event_from_room_row(row: sqlite3.Row) -> dict[str, Any]:
return {
"room_id": str(row["room_id"]), "seq": int(row["seq"]), "event_id": str(row["event_id"]),
"kind": str(row["kind"]), "actor": json.loads(row["actor_json"]),
"authority_epoch": row["authority_epoch"], "payload": json.loads(row["payload_json"]),
"created_at": float(row["created_at"]), "idempotent": False,
}
_event_from_room_row = hosted_rooms._event_from_row
def _require_room(conn: sqlite3.Connection, room_id: str) -> None:
@@ -94,17 +85,14 @@ def _require_room(conn: sqlite3.Connection, room_id: str) -> None:
def _settled_message(
conn: sqlite3.Connection, room_id: str, discussion_event_id: str, message_event_id: Any
) -> dict[str, Any] | None:
conn: sqlite3.Connection, room_id: str, discussion_event_id: str, message_event_id: Any) -> dict[str, Any] | None:
"""Return the indexed member message a ``turn.settled`` event committed, if it is in the projection."""
rows = conn.execute(
"SELECT seq, event_json FROM hosted_room_policy_events WHERE room_id=? AND discussion_event_id=?",
(room_id, discussion_event_id),
).fetchall()
(room_id, discussion_event_id)).fetchall()
return next(
(m for m in (json.loads(row["event_json"]) for row in rows) if m.get("event_id") == message_event_id),
None,
)
None)
class HostedRoomPolicyCheckpoint:
@@ -127,43 +115,33 @@ class HostedRoomPolicyCheckpoint:
@staticmethod
def _store_active_event(
conn: sqlite3.Connection, *, event: Mapping[str, Any], thread_id: str, discussion_event_id: str
) -> None:
conn: sqlite3.Connection, *, event: Mapping[str, Any], thread_id: str, discussion_event_id: str) -> None:
conn.execute(
"""INSERT OR IGNORE INTO hosted_room_policy_events(
room_id, thread_id, discussion_event_id, seq, event_json
) VALUES (?, ?, ?, ?, ?)""",
(
event["room_id"], thread_id, discussion_event_id, int(event["seq"]),
json.dumps(dict(event), ensure_ascii=True, sort_keys=True, separators=(",", ":")),
),
)
(event["room_id"], thread_id, discussion_event_id, int(event["seq"]), compact_json(dict(event))))
@staticmethod
def _store_transcript_event(
conn: sqlite3.Connection, *, event: Mapping[str, Any], thread_id: str,
settled_seq: int | None = None,
) -> None:
conn: sqlite3.Connection, *, event: Mapping[str, Any], thread_id: str, settled_seq: int | None = None) -> None:
conn.execute(
"""INSERT INTO hosted_room_policy_transcript(
room_id, thread_id, seq, kind, settled_seq
) VALUES (?, ?, ?, ?, ?)
ON CONFLICT(room_id, thread_id, seq) DO UPDATE SET
settled_seq=COALESCE(excluded.settled_seq, hosted_room_policy_transcript.settled_seq)""",
(event["room_id"], thread_id, int(event["seq"]), str(event["kind"]), settled_seq),
)
(event["room_id"], thread_id, int(event["seq"]), str(event["kind"]), settled_seq))
if event["kind"] in {"message.user", "message.member"}:
cutoff = conn.execute(
"""SELECT seq FROM hosted_room_policy_transcript
WHERE room_id=? AND thread_id=? AND kind IN ('message.user', 'message.member')
ORDER BY seq DESC LIMIT 1 OFFSET ?""",
(event["room_id"], thread_id, MAX_THREAD_TRANSCRIPT_EVENTS - 1),
).fetchone()
(event["room_id"], thread_id, MAX_THREAD_TRANSCRIPT_EVENTS - 1)).fetchone()
if cutoff is not None:
conn.execute(
"DELETE FROM hosted_room_policy_transcript WHERE room_id=? AND thread_id=? AND seq<?",
(event["room_id"], thread_id, int(cutoff["seq"])),
)
(event["room_id"], thread_id, int(cutoff["seq"])))
def _backfill_transcript(self, conn: sqlite3.Connection, *, room_id: str, through_seq: int) -> None:
"""Migrate bounded committed thread history from the durable room log."""
@@ -172,9 +150,7 @@ class HostedRoomPolicyCheckpoint:
settled_seq_by_message: dict[str, int] = {}
for row in conn.execute(
"""SELECT seq, payload_json FROM hosted_room_events
WHERE room_id=? AND seq<=? AND kind='turn.settled' ORDER BY seq""",
(room_id, through_seq),
):
WHERE room_id=? AND seq<=? AND kind='turn.settled' ORDER BY seq""", (room_id, through_seq)):
message_event_id = str(json.loads(row["payload_json"]).get("message_event_id") or "")
if message_event_id:
settled_seq_by_message[message_event_id] = int(row["seq"])
@@ -182,8 +158,7 @@ class HostedRoomPolicyCheckpoint:
f"""SELECT {_ROOM_EVENT_COLUMNS} FROM hosted_room_events
WHERE room_id=? AND seq<=? AND kind IN ('message.user', 'message.member')
ORDER BY seq""",
(room_id, through_seq),
)
(room_id, through_seq))
for row in rows:
if row["kind"] == "message.member" and row["event_id"] not in settled_seq_by_message:
continue
@@ -192,38 +167,30 @@ class HostedRoomPolicyCheckpoint:
if thread_id:
self._store_transcript_event(
conn, event=event, thread_id=thread_id,
settled_seq=settled_seq_by_message.get(str(row["event_id"])),
)
settled_seq=settled_seq_by_message.get(str(row["event_id"])))
def _discussion_events(
self, conn: sqlite3.Connection, *, room_id: str, thread_id: str, discussion_event_id: str,
bound_error: str,
) -> list[dict[str, Any]]:
bound_error: str) -> list[dict[str, Any]]:
"""Merge the thread transcript with the active projection, ordered by seq."""
active_rows = conn.execute(
"""SELECT event_json FROM hosted_room_policy_events
WHERE room_id=? AND discussion_event_id=? ORDER BY seq LIMIT ?""",
(room_id, discussion_event_id, MAX_ACTIVE_POLICY_EVENTS + 1),
).fetchall()
(room_id, discussion_event_id, MAX_ACTIVE_POLICY_EVENTS + 1)).fetchall()
if len(active_rows) > MAX_ACTIVE_POLICY_EVENTS:
raise RuntimeError(bound_error)
rows = conn.execute(
_TRANSCRIPT_EVENTS_SQL, (room_id, thread_id, room_id, thread_id, room_id)
).fetchall()
rows = conn.execute(_TRANSCRIPT_EVENTS_SQL, (room_id, thread_id, room_id, thread_id, room_id)).fetchall()
events_by_seq = {
int(event["seq"]): event
for event in (
*(_event_from_room_row(row) for row in rows),
*(json.loads(row["event_json"]) for row in active_rows),
)
}
*(json.loads(row["event_json"]) for row in active_rows))}
return [events_by_seq[seq] for seq in sorted(events_by_seq)]
# -- per-kind projection handlers (dispatched by _apply_event) -----------
def _apply_user_message(
self, conn: sqlite3.Connection, event: Mapping[str, Any], payload: Mapping[str, Any]
) -> None:
self, conn: sqlite3.Connection, event: Mapping[str, Any], payload: Mapping[str, Any]) -> None:
room_id = str(event["room_id"])
thread_id = str(payload.get("thread_id") or "")
event_id = str(event.get("event_id") or "")
@@ -236,14 +203,12 @@ class HostedRoomPolicyCheckpoint:
ON CONFLICT(room_id, thread_id) DO UPDATE SET
discussion_event_id=excluded.discussion_event_id,
latest_user_seq=excluded.latest_user_seq, completed=0""",
(room_id, thread_id, event_id, int(event["seq"])),
)
(room_id, thread_id, event_id, int(event["seq"])))
self._store_active_event(conn, event=event, thread_id=thread_id, discussion_event_id=event_id)
self._store_transcript_event(conn, event=event, thread_id=thread_id)
def _apply_discussion_event(
self, conn: sqlite3.Connection, event: Mapping[str, Any], payload: Mapping[str, Any]
) -> None:
self, conn: sqlite3.Connection, event: Mapping[str, Any], payload: Mapping[str, Any]) -> None:
"""Index member messages and terminal turn outcomes of a known discussion."""
room_id = str(event["room_id"])
seq = int(event["seq"])
@@ -252,26 +217,20 @@ class HostedRoomPolicyCheckpoint:
discussion_event_id = str(payload.get("discussion_event_id") or "")
source = conn.execute(
"SELECT 1 FROM hosted_room_policy_events WHERE room_id=? AND discussion_event_id=? LIMIT 1",
(room_id, discussion_event_id),
).fetchone()
(room_id, discussion_event_id)).fetchone()
if source is None:
return
self._store_active_event(
conn, event=event, thread_id=thread_id, discussion_event_id=discussion_event_id
)
self._store_active_event(conn, event=event, thread_id=thread_id, discussion_event_id=discussion_event_id)
if kind not in _TERMINAL_KINDS:
return
task_id = str(payload.get("task_id") or "")
execution_generation = (
int(payload.get("execution_generation") or 0) if kind == "turn.deferred" else 0
)
execution_generation = (int(payload.get("execution_generation") or 0) if kind == "turn.deferred" else 0)
if task_id:
conn.execute(
"""INSERT OR IGNORE INTO hosted_room_policy_publications(
room_id, task_id, kind, execution_generation, seq
) VALUES (?, ?, ?, ?, ?)""",
(room_id, task_id, kind, execution_generation, seq),
)
(room_id, task_id, kind, execution_generation, seq))
member_id = str(payload.get("member_id") or "")
seen_through_seq = int(payload.get("seen_through_seq") or 0)
if kind == "turn.settled" and payload.get("message_event_id"):
@@ -286,35 +245,26 @@ class HostedRoomPolicyCheckpoint:
) VALUES (?, ?, ?, ?)
ON CONFLICT(room_id, thread_id, member_id) DO UPDATE SET
seen_through_seq=MAX(hosted_room_policy_watermarks.seen_through_seq, excluded.seen_through_seq)""",
(room_id, thread_id, member_id, seen_through_seq),
)
(room_id, thread_id, member_id, seen_through_seq))
def _apply_room_activity(
self, conn: sqlite3.Connection, event: Mapping[str, Any], payload: Mapping[str, Any]
) -> None:
self, conn: sqlite3.Connection, event: Mapping[str, Any], payload: Mapping[str, Any]) -> None:
room_id = str(event["room_id"])
thread_id = str(payload.get("thread_id") or "")
discussion_event_id = str(payload.get("discussion_event_id") or "")
conn.execute(_DELETE_ACTIVE_EVENTS_SQL, (room_id, discussion_event_id))
conn.execute(
"DELETE FROM hosted_room_policy_threads WHERE room_id=? AND thread_id=?",
(room_id, thread_id),
)
conn.execute("DELETE FROM hosted_room_policy_threads WHERE room_id=? AND thread_id=?", (room_id, thread_id))
def _apply_stop_requested(
self, conn: sqlite3.Connection, event: Mapping[str, Any], payload: Mapping[str, Any]
) -> None:
self, conn: sqlite3.Connection, event: Mapping[str, Any], payload: Mapping[str, Any]) -> None:
conn.execute(
"""UPDATE hosted_room_policy_cursors
SET stopped_through_seq=MAX(stopped_through_seq, ?) WHERE room_id=?""",
(int(event["seq"]), str(event["room_id"])),
)
(int(event["seq"]), str(event["room_id"])))
_APPLY_BY_KIND: dict[str, Callable[..., None]] = {
"message.user": _apply_user_message,
"message.member": _apply_discussion_event,
**dict.fromkeys(_TERMINAL_KINDS, _apply_discussion_event),
"room.activity": _apply_room_activity,
"message.user": _apply_user_message, "message.member": _apply_discussion_event,
**dict.fromkeys(_TERMINAL_KINDS, _apply_discussion_event), "room.activity": _apply_room_activity,
"room.stop_requested": _apply_stop_requested,
}
@@ -331,25 +281,18 @@ class HostedRoomPolicyCheckpoint:
conn.execute(
"""INSERT OR IGNORE INTO hosted_room_policy_cursors(
room_id, through_seq, stopped_through_seq, updated_at
) VALUES (?, 0, 0, 0)""",
(room_id,),
)
row = conn.execute(
"SELECT through_seq FROM hosted_room_policy_cursors WHERE room_id=?", (room_id,)
).fetchone()
) VALUES (?, 0, 0, 0)""", (room_id,))
row = conn.execute("SELECT through_seq FROM hosted_room_policy_cursors WHERE room_id=?", (room_id,)).fetchone()
cursor = int(row["through_seq"])
transcript_state = conn.execute(
"SELECT schema_version FROM hosted_room_policy_transcript_state WHERE room_id=?",
(room_id,),
).fetchone()
"SELECT schema_version FROM hosted_room_policy_transcript_state WHERE room_id=?", (room_id,)).fetchone()
if transcript_state is None or int(transcript_state["schema_version"]) < _TRANSCRIPT_SCHEMA_VERSION:
self._backfill_transcript(conn, room_id=room_id, through_seq=cursor)
conn.execute(
"""INSERT INTO hosted_room_policy_transcript_state(room_id, schema_version)
VALUES (?, ?)
ON CONFLICT(room_id) DO UPDATE SET schema_version=excluded.schema_version""",
(room_id, _TRANSCRIPT_SCHEMA_VERSION),
)
(room_id, _TRANSCRIPT_SCHEMA_VERSION))
return cursor
def sync(self, *, room_id: str, latest_seq: int) -> int:
@@ -362,8 +305,7 @@ class HostedRoomPolicyCheckpoint:
while cursor < latest_seq:
page = hosted_rooms.read_events(
self.db_path, room_id=room_id, since_seq=cursor, limit=hosted_rooms.MAX_LOG_LIMIT
)
self.db_path, room_id=room_id, since_seq=cursor, limit=hosted_rooms.MAX_LOG_LIMIT)
rows = [event for event in page.get("events", []) if isinstance(event, Mapping)]
next_cursor = int(page.get("cursor") or cursor)
if not rows or next_cursor <= cursor:
@@ -375,8 +317,7 @@ class HostedRoomPolicyCheckpoint:
self._apply_event(conn, event)
updated = conn.execute(
"UPDATE hosted_room_policy_cursors SET through_seq=?, updated_at=? WHERE room_id=?",
(next_cursor, float(rows[-1].get("created_at") or 0), room_id),
)
(next_cursor, float(rows[-1].get("created_at") or 0), room_id))
if updated.rowcount != 1:
raise RuntimeError("room policy cursor disappeared during replay")
cursor = next_cursor
@@ -387,43 +328,29 @@ class HostedRoomPolicyCheckpoint:
through_seq = self.sync(room_id=room_id, latest_seq=latest_seq)
with self._connect() as conn:
cursor = conn.execute(
"SELECT stopped_through_seq FROM hosted_room_policy_cursors WHERE room_id=?",
(room_id,),
).fetchone()
"SELECT stopped_through_seq FROM hosted_room_policy_cursors WHERE room_id=?", (room_id,)).fetchone()
stopped_through_seq = int(cursor["stopped_through_seq"])
thread = conn.execute(
"""SELECT thread_id, discussion_event_id FROM hosted_room_policy_threads
WHERE room_id=? AND completed=0 AND latest_user_seq>?
ORDER BY latest_user_seq, thread_id LIMIT 1""",
(room_id, stopped_through_seq),
).fetchone()
ORDER BY latest_user_seq, thread_id LIMIT 1""", (room_id, stopped_through_seq)).fetchone()
if thread is None:
return PolicySnapshot(
through_seq=through_seq, stopped_through_seq=stopped_through_seq,
events=(), watermarks={},
)
events=(), watermarks={})
thread_id = str(thread["thread_id"])
events = self._discussion_events(
conn, room_id=room_id, thread_id=thread_id,
discussion_event_id=str(thread["discussion_event_id"]),
bound_error="active room policy projection exceeded its bound",
)
bound_error="active room policy projection exceeded its bound")
watermark_rows = conn.execute(
"""SELECT member_id, seen_through_seq FROM hosted_room_policy_watermarks
WHERE room_id=? AND thread_id=?""",
(room_id, thread_id),
).fetchall()
WHERE room_id=? AND thread_id=?""", (room_id, thread_id)).fetchall()
return PolicySnapshot(
through_seq=through_seq, stopped_through_seq=stopped_through_seq, events=tuple(events),
watermarks={
(thread_id, str(row["member_id"])): int(row["seen_through_seq"])
for row in watermark_rows
},
)
watermarks={(thread_id, str(row["member_id"])): int(row["seen_through_seq"]) for row in watermark_rows})
def publication_exists(
self, *, room_id: str, task_id: str, status: str, execution_generation: int
) -> bool:
def publication_exists(self, *, room_id: str, task_id: str, status: str, execution_generation: int) -> bool:
"""Return whether one exact driver outcome is already in the room log."""
if status == "deferred":
sql = """SELECT 1 FROM hosted_room_policy_publications
@@ -441,25 +368,20 @@ class HostedRoomPolicyCheckpoint:
with self._connect() as conn:
source = conn.execute(
"SELECT discussion_event_id, thread_id FROM hosted_room_policy_events WHERE room_id=? AND seq=?",
(room_id, source_event_seq),
).fetchone()
(room_id, source_event_seq)).fetchone()
if source is None:
return []
return self._discussion_events(
conn, room_id=room_id, thread_id=str(source["thread_id"]),
discussion_event_id=str(source["discussion_event_id"]),
bound_error="task policy projection exceeded its bound",
)
bound_error="task policy projection exceeded its bound")
def compact_completed(self, *, room_id: str) -> None:
"""Drop any completed projections left by an interrupted sync."""
with self._connect() as conn:
completed = conn.execute(
"SELECT discussion_event_id FROM hosted_room_policy_threads WHERE room_id=? AND completed=1",
(room_id,),
).fetchall()
(room_id,)).fetchall()
for row in completed:
conn.execute(_DELETE_ACTIVE_EVENTS_SQL, (room_id, str(row["discussion_event_id"])))
conn.execute(
"DELETE FROM hosted_room_policy_threads WHERE room_id=? AND completed=1", (room_id,)
)
conn.execute("DELETE FROM hosted_room_policy_threads WHERE room_id=? AND completed=1", (room_id,))
gateway/hosted_room_replicas.py
+58 -124
View File
@@ -1,41 +1,35 @@
"""Replica store and takeover primitives for hosted Group Chat rooms.
The authority gateway owns a room's ordered log in ``gateway/hosted_rooms.py``;
this module gives every OTHER participant gateway a durable local copy plus the
fenced primitives to continue the room when the authority host dies:
``ingest_page()`` (idempotent, gap- and epoch-regression-safe replay),
``promote_replica()`` (resume locally at ``epoch + 1`` with a lineage-proving
``authority.claimed`` event) and ``demote_room()`` (a returning stale authority
records ``authority.lost`` when shown proof of a newer epoch).
Storage primitives only: none decide *when* takeover is safe. The caller must
establish that the previous owner can no longer commit before promoting.
Non-authority gateways keep a durable local copy of the room log (``ingest_page()``: idempotent,
gap- and epoch-regression-safe) plus fenced primitives to continue the room when the authority host
dies: ``promote_replica()`` resumes locally at ``epoch + 1`` with a lineage-proving ``authority.claimed``
event; ``demote_room()`` records ``authority.lost`` when a returning stale authority is shown a newer
epoch. Storage primitives only: the caller decides *when* takeover is safe.
"""
from __future__ import annotations
import json
import sqlite3
import time
from contextlib import contextmanager
from functools import partial
from pathlib import Path
from typing import Any, Iterator
from gateway.hosted_rooms import (
MAX_ACTOR_ID_CHARS,
MAX_EVENT_JSON_BYTES,
MAX_ROOM_ID_CHARS,
HostedRoomError,
RoomConflictError,
_canonical_json,
_actor_json,
_connect,
_payload_json,
_room_id,
_transaction,
_validate_identifier,
_validate_members,
_validate_room_name,
local_authority_gateway_id,
)
from gateway.hosted_rooms_common import positive_int, utf8_len
local_authority_gateway_id)
from gateway.hosted_rooms_common import bounded_int, clock, utf8_len
MAX_REPLICA_ROOMS = 256
MAX_REPLICA_EVENT_BYTES = 256 * 1024 * 1024
@@ -46,16 +40,13 @@ _INSERT_REPLICA_EVENT = f"INSERT INTO hosted_room_replica_events {_EVENT_COLUMNS
_SELECT_REPLICA = "SELECT * FROM hosted_room_replicas WHERE room_id=?"
class ReplicaError(HostedRoomError):
"""Base class for invalid or conflicting replica operations."""
class ReplicaError(HostedRoomError): """Base class for invalid or conflicting replica operations."""
class ReplicaGapError(ReplicaError):
"""A page does not start at the replica's next expected sequence."""
class ReplicaGapError(ReplicaError): """A page does not start at the replica's next expected sequence."""
class ReplicaEpochRegressionError(ReplicaError):
"""A page or demotion carries an older authority epoch than stored."""
class ReplicaEpochRegressionError(ReplicaError): """A page or demotion carries an older authority epoch than stored."""
def _initialize_replica_schema(conn: sqlite3.Connection) -> None:
@@ -83,8 +74,7 @@ def _initialize_replica_schema(conn: sqlite3.Connection) -> None:
def _replica_transaction(db_path: Path | str) -> Iterator[sqlite3.Connection]:
"""Ensure the replica schema (own autocommit connection), then open an IMMEDIATE transaction.
The schema DDL is re-run inside the transaction too; that double init is the
established statement order and is intentionally preserved.
The DDL is deliberately re-run inside the transaction: that double init is the established statement order.
"""
conn = _connect(db_path)
try:
@@ -97,28 +87,19 @@ def _replica_transaction(db_path: Path | str) -> Iterator[sqlite3.Connection]:
yield conn
def _room_id(value: Any) -> str:
return _validate_identifier(value, label="room_id", max_chars=MAX_ROOM_ID_CHARS)
def _positive_int(value: Any, message: str) -> int:
return positive_int(value, error=ReplicaError, message=message)
_positive_int = partial(bounded_int, error=ReplicaError, low=1)
def _control_event_json(payload: dict[str, Any]) -> tuple[str, str]:
"""Canonical (actor_json, payload_json) for a system authority-control event."""
return (
_canonical_json(_SYSTEM_ACTOR, label="actor", max_bytes=4 * 1024),
_canonical_json(payload, label="payload", max_bytes=MAX_EVENT_JSON_BYTES),
)
return _actor_json(_SYSTEM_ACTOR), _payload_json(payload)
def _event_bytes(event: dict[str, Any]) -> int:
return utf8_len(
str(event["event_id"]), str(event["kind"]),
json.dumps(event["actor"], ensure_ascii=False, separators=(",", ":")),
json.dumps(event["payload"], ensure_ascii=False, separators=(",", ":")),
)
json.dumps(event["payload"], ensure_ascii=False, separators=(",", ":")))
def _validate_page(page: Any) -> tuple[list[dict[str, Any]], dict[str, Any]]:
@@ -131,14 +112,13 @@ def _validate_page(page: Any) -> tuple[list[dict[str, Any]], dict[str, Any]]:
if not isinstance(authority, dict):
raise ReplicaError("page.authority is required for replication")
gateway_id = _validate_identifier(
authority.get("gateway_id"), label="page.authority.gateway_id", max_chars=MAX_ACTOR_ID_CHARS
)
epoch = _positive_int(authority.get("epoch"), "page.authority.epoch must be a positive integer")
authority.get("gateway_id"), label="page.authority.gateway_id", max_chars=MAX_ACTOR_ID_CHARS)
epoch = _positive_int(authority.get("epoch"), message="page.authority.epoch must be a positive integer")
previous_seq: int | None = None
for event in events:
if not isinstance(event, dict):
raise ReplicaError("page events must be objects")
seq = _positive_int(event.get("seq"), "event.seq must be a positive integer")
seq = _positive_int(event.get("seq"), message="event.seq must be a positive integer")
if previous_seq is not None and seq != previous_seq + 1:
raise ReplicaGapError("page events must be contiguous")
previous_seq = seq
@@ -156,9 +136,7 @@ def _replica_row_state(conn: sqlite3.Connection, room_id: str) -> tuple[sqlite3.
"""Return (row, stored_epoch, last_seq, stored_bytes); a new room is admitted only under the room cap."""
row = conn.execute(
"""SELECT authority_gateway_id, authority_epoch, last_seq, latest_seq, event_bytes
FROM hosted_room_replicas WHERE room_id=?""",
(room_id,),
).fetchone()
FROM hosted_room_replicas WHERE room_id=?""", (room_id,)).fetchone()
if row is None:
count = conn.execute("SELECT COUNT(*) FROM hosted_room_replicas").fetchone()[0]
if int(count) >= MAX_REPLICA_ROOMS:
@@ -169,8 +147,7 @@ def _replica_row_state(conn: sqlite3.Connection, room_id: str) -> tuple[sqlite3.
def _store_replica(
conn: sqlite3.Connection, *, is_new: bool, room_id: str, room_name: str, members_json: str,
authority: dict[str, Any], new_last: int, latest_seq: int, added_bytes: int, now: float,
) -> None:
authority: dict[str, Any], new_last: int, latest_seq: int, added_bytes: int, now: float) -> None:
"""INSERT the replica row for a new room, else UPDATE it (event_bytes accumulates)."""
values = (room_name, members_json, authority["gateway_id"], authority["epoch"], new_last, max(latest_seq, new_last))
if is_new:
@@ -178,30 +155,23 @@ def _store_replica(
"""INSERT INTO hosted_room_replicas (room_id, name, members_json,
authority_gateway_id, authority_epoch, last_seq, latest_seq, event_bytes,
created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
(room_id, *values, added_bytes, now, now),
)
(room_id, *values, added_bytes, now, now))
else:
conn.execute(
"""UPDATE hosted_room_replicas SET name=?, members_json=?, authority_gateway_id=?,
authority_epoch=?, last_seq=?, latest_seq=?, event_bytes=event_bytes+?,
updated_at=? WHERE room_id=?""",
(*values, added_bytes, now, room_id),
)
updated_at=? WHERE room_id=?""", (*values, added_bytes, now, room_id))
def ingest_page(
db_path: Path | str, *, room_id: Any, room_name: Any, members: Any, page: Any, now: float | None = None
) -> dict[str, Any]:
"""Persist one replay page idempotently; refuses seq gaps and epoch regressions.
``page`` is the verbatim ``groups.log`` (``read_events()``) result whose
``authority`` stamp proves lineage.
"""
"""Persist one verbatim ``read_events()`` page idempotently; refuses seq gaps and epoch regressions."""
room_id = _room_id(room_id)
room_name = _validate_room_name(room_name)
_, members_json = _validate_members(members)
events, authority = _validate_page(page)
now = time.time() if now is None else float(now)
now = clock(now)
with _replica_transaction(db_path) as conn:
row, stored_epoch, last_seq, stored_bytes = _replica_row_state(conn, room_id)
@@ -216,15 +186,13 @@ def ingest_page(
size = _event_bytes(event)
if stored_bytes + added_bytes + size > MAX_REPLICA_EVENT_BYTES:
raise ReplicaError("replica event storage exhausted")
actor_json = _canonical_json(event["actor"], label="actor", max_bytes=4 * 1024)
payload_json = _canonical_json(event["payload"], label="payload", max_bytes=MAX_EVENT_JSON_BYTES)
actor_json = _actor_json(event["actor"])
payload_json = _payload_json(event["payload"])
conn.execute(
_INSERT_REPLICA_EVENT,
(
room_id, int(event["seq"]), event["event_id"], event["kind"], actor_json,
event.get("authority_epoch"), payload_json, float(event.get("created_at") or now),
),
)
event.get("authority_epoch"), payload_json, float(event.get("created_at") or now)))
added_bytes += size
new_last = int(new_events[-1]["seq"]) if new_events else last_seq
latest_seq = page.get("latest_seq")
@@ -232,13 +200,9 @@ def ingest_page(
latest_seq = new_last
_store_replica(
conn, is_new=row is None, room_id=room_id, room_name=room_name, members_json=members_json,
authority=authority, new_last=new_last, latest_seq=latest_seq, added_bytes=added_bytes, now=now,
)
authority=authority, new_last=new_last, latest_seq=latest_seq, added_bytes=added_bytes, now=now)
return {
"room_id": room_id,
"stored_seq": new_last,
"ingested": len(new_events),
"authority": authority,
"room_id": room_id, "stored_seq": new_last, "ingested": len(new_events), "authority": authority,
"caught_up": new_last >= max(latest_seq, new_last),
}
@@ -251,15 +215,10 @@ def replica_state(db_path: Path | str, *, room_id: Any) -> dict[str, Any]:
if row is None:
raise ReplicaError("replica not found")
return {
"room_id": row["room_id"],
"name": row["name"],
"members": json.loads(row["members_json"]),
"room_id": row["room_id"], "name": row["name"], "members": json.loads(row["members_json"]),
"authority": {"gateway_id": row["authority_gateway_id"], "epoch": int(row["authority_epoch"])},
"last_seq": int(row["last_seq"]),
"latest_seq": int(row["latest_seq"]),
"event_bytes": int(row["event_bytes"]),
"created_at": float(row["created_at"]),
"updated_at": float(row["updated_at"]),
"last_seq": int(row["last_seq"]), "latest_seq": int(row["latest_seq"]), "event_bytes": int(row["event_bytes"]),
"created_at": float(row["created_at"]), "updated_at": float(row["updated_at"]),
}
@@ -268,15 +227,14 @@ def promote_replica(
) -> dict[str, Any]:
"""Continue a replicated room on THIS gateway at ``epoch + 1``.
Copies the replica log into the authoritative store and appends a
lineage-proving ``authority.claimed`` event; wherever the claim replicates,
the old epoch is stale and every fenced primitive rejects it. The caller
decides takeover is safe; this only makes it atomic and provable.
Copies the replica log into the authoritative store and appends a lineage-proving
``authority.claimed`` event, so wherever the claim replicates the old epoch is stale and every
fenced primitive rejects it. The caller decides takeover is safe; this makes it atomic and provable.
"""
room_id = _room_id(room_id)
if not isinstance(reason, str) or not reason or len(reason) > 200:
raise ReplicaError("reason must be a non-empty string of at most 200 chars")
now = time.time() if now is None else float(now)
now = clock(now)
local_gateway = local_authority_gateway_id()
with _replica_transaction(db_path) as conn:
@@ -296,11 +254,8 @@ def promote_replica(
claim_seq = int(replica["last_seq"]) + 1
claim_event_id = f"system:authority-claimed:{target_epoch}"
claim_actor_json, claim_payload_json = _control_event_json({
"previous_gateway_id": previous_gateway,
"authority_gateway_id": local_gateway,
"authority_epoch": target_epoch,
"promoted_from_replica": True,
"reason": reason,
"previous_gateway_id": previous_gateway, "authority_gateway_id": local_gateway,
"authority_epoch": target_epoch, "promoted_from_replica": True, "reason": reason,
})
claim_bytes = utf8_len(claim_event_id, "authority.claimed", claim_actor_json, claim_payload_json)
@@ -311,31 +266,22 @@ def promote_replica(
VALUES (?, ?, ?, ?, ?, ?, ?, 1, ?, ?, NULL)""",
(
room_id, replica["name"], replica["members_json"], local_gateway, target_epoch,
claim_seq + 1, int(replica["event_bytes"]) + claim_bytes, now, now,
),
)
claim_seq + 1, int(replica["event_bytes"]) + claim_bytes, now, now))
conn.execute(
f"""INSERT INTO hosted_room_events {_EVENT_COLUMNS}
SELECT room_id, seq, event_id, kind, actor_json, authority_epoch, payload_json, created_at
FROM hosted_room_replica_events WHERE room_id=?""",
(room_id,),
)
(room_id,))
conn.execute(
_INSERT_ROOM_EVENT,
(
room_id, claim_seq, claim_event_id, "authority.claimed",
claim_actor_json, target_epoch, claim_payload_json, now,
),
)
claim_actor_json, target_epoch, claim_payload_json, now))
conn.execute("DELETE FROM hosted_room_replica_events WHERE room_id=?", (room_id,))
conn.execute("DELETE FROM hosted_room_replicas WHERE room_id=?", (room_id,))
return {
"room_id": room_id,
"authority_gateway_id": local_gateway,
"authority_epoch": target_epoch,
"previous_gateway_id": previous_gateway,
"previous_epoch": previous_epoch,
"claim_seq": claim_seq,
"room_id": room_id, "authority_gateway_id": local_gateway, "authority_epoch": target_epoch,
"previous_gateway_id": previous_gateway, "previous_epoch": previous_epoch, "claim_seq": claim_seq,
"latest_seq": claim_seq,
}
@@ -345,34 +291,28 @@ def demote_room(
) -> dict[str, Any]:
"""Fence THIS gateway's stale room authority against a proven newer epoch.
Called when a returning gateway observes (replicated ``authority.claimed``
or a transport rejection) that another gateway owns the room at a higher
epoch. Appends ``authority.lost`` and adopts the observed lineage so no
local send can commit at the stale epoch. Idempotent per lineage.
When a returning gateway observes (replicated ``authority.claimed`` or a transport rejection) that
another gateway owns the room at a higher epoch, append ``authority.lost`` and adopt the observed
lineage so no local send can commit at the stale epoch. Idempotent per lineage.
"""
room_id = _room_id(room_id)
observed_gateway_id = _validate_identifier(
observed_gateway_id, label="observed_gateway_id", max_chars=MAX_ACTOR_ID_CHARS
)
observed_epoch = _positive_int(observed_epoch, "observed_epoch must be a positive integer")
now = time.time() if now is None else float(now)
observed_gateway_id, label="observed_gateway_id", max_chars=MAX_ACTOR_ID_CHARS)
observed_epoch = _positive_int(observed_epoch, message="observed_epoch must be a positive integer")
now = clock(now)
local_gateway = local_authority_gateway_id()
with _transaction(db_path, immediate=True) as conn:
row = conn.execute(
"""SELECT authority_gateway_id, authority_epoch, next_seq
FROM hosted_rooms WHERE room_id=? AND disbanded_at IS NULL""",
(room_id,),
).fetchone()
FROM hosted_rooms WHERE room_id=? AND disbanded_at IS NULL""", (room_id,)).fetchone()
if row is None:
raise ReplicaError("room not found in the local authoritative store")
current_gateway = str(row["authority_gateway_id"])
current_epoch = int(row["authority_epoch"])
if current_gateway == observed_gateway_id and current_epoch == observed_epoch:
return {
"room_id": room_id,
"authority_gateway_id": current_gateway,
"authority_epoch": current_epoch,
"room_id": room_id, "authority_gateway_id": current_gateway, "authority_epoch": current_epoch,
"idempotent": True,
}
if observed_epoch <= current_epoch:
@@ -382,24 +322,18 @@ def demote_room(
lost_actor_json, lost_payload_json = _control_event_json({
"previous_gateway_id": current_gateway,
"authority_gateway_id": observed_gateway_id,
"authority_epoch": observed_epoch,
})
"authority_epoch": observed_epoch})
conn.execute(
_INSERT_ROOM_EVENT,
(
room_id, int(row["next_seq"]), f"system:authority-lost:{observed_epoch}",
"authority.lost", lost_actor_json, observed_epoch, lost_payload_json, now,
),
)
"authority.lost", lost_actor_json, observed_epoch, lost_payload_json, now))
conn.execute(
"""UPDATE hosted_rooms
SET authority_gateway_id=?, authority_epoch=?, next_seq=next_seq+1, revision=revision+1, updated_at=?
WHERE room_id=?""",
(observed_gateway_id, observed_epoch, now, room_id),
)
(observed_gateway_id, observed_epoch, now, room_id))
return {
"room_id": room_id,
"authority_gateway_id": observed_gateway_id,
"authority_epoch": observed_epoch,
"room_id": room_id, "authority_gateway_id": observed_gateway_id, "authority_epoch": observed_epoch,
"idempotent": False,
}
gateway/hosted_rooms.py
+272 -601
View File
File diff suppressed because it is too large Load Diff
gateway/hosted_rooms_common.py
+84 -51
View File
@@ -1,11 +1,9 @@
"""Shared leaf helpers for the gateway hosted-room modules.
Every hosted-room module validates identifiers, bounded integers, exact field
sets and canonical JSON with its own error class and its own error strings
(tests pin those strings). These helpers take the error class and message
templates as parameters so each caller keeps byte-identical failures while the
logic lives once. This module must stay a leaf: never import a hosted_room*
origin module from here (import cycle).
Each hosted-room module validates identifiers, bounded integers, exact field sets and
canonical JSON with its own error class and pinned error strings; these helpers take the
error class and message templates as parameters so failures stay byte-identical while the
logic lives once. Must stay a leaf: never import a hosted_room* module (import cycle).
"""
from __future__ import annotations
@@ -13,7 +11,9 @@ from __future__ import annotations
import json
import re
import sqlite3
import time
from contextlib import contextmanager
from functools import partial
from pathlib import Path
from typing import Any, Callable, Iterator, Mapping
@@ -21,47 +21,33 @@ IDENTIFIER_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]*$")
def identifier(
value: Any,
*,
label: str,
error: type[Exception],
max_chars: int = 128,
pattern: re.Pattern[str] = IDENTIFIER_RE,
invalid: str | None = None,
) -> str:
"""Strip and validate a bounded identifier; ``invalid`` overrides the fail message."""
value: Any, *, label: str, error: type[Exception], max_chars: int = 128,
pattern: re.Pattern[str] | None = IDENTIFIER_RE, invalid: str | None = None) -> str:
"""Strip and validate a bounded string; ``pattern=None`` skips the shape check."""
if not isinstance(value, str):
raise error(f"{label} must be a string")
value = value.strip()
if not value or len(value) > max_chars or not pattern.fullmatch(value):
if not value or len(value) > max_chars or (pattern is not None and not pattern.fullmatch(value)):
raise error(invalid or f"invalid {label}")
return value
def positive_int(value: Any, *, error: type[Exception], message: str) -> int:
"""Reject bools, non-ints and values below 1 (``message`` is the exact error text)."""
if isinstance(value, bool) or not isinstance(value, int) or value < 1:
def bounded_int(value: Any, *, error: type[Exception], message: str, low: int = 0, high: int | None = None) -> int:
"""Reject bools, non-ints and values outside ``[low, high]`` (``message`` is the exact text)."""
if isinstance(value, bool) or not isinstance(value, int) or value < low or (high is not None and value > high):
raise error(message)
return value
def non_negative_int(value: Any, *, error: type[Exception], message: str) -> int:
if isinstance(value, bool) or not isinstance(value, int) or value < 0:
raise error(message)
return value
positive_int = partial(bounded_int, low=1)
non_negative_int = partial(bounded_int, low=0)
def exact_fields(
value: Any,
*,
label: str,
required: frozenset[str] | set[str],
optional: frozenset[str] | set[str] = frozenset(),
error: type[Exception],
not_object: str | None = None,
value: Any, *, label: str, required: frozenset[str] | set[str],
optional: frozenset[str] | set[str] = frozenset(), error: type[Exception], not_object: str | None = None,
missing_fmt: str = "{label} is missing fields: {fields}",
unknown_fmt: str = "{label} has unknown fields: {fields}",
) -> Mapping[str, Any]:
unknown_fmt: str = "{label} has unknown fields: {fields}") -> Mapping[str, Any]:
"""Require exactly ``required`` (+ any ``optional``) keys; formats name the offenders sorted."""
if not isinstance(value, Mapping):
raise error(not_object or f"{label} must be an object")
@@ -75,19 +61,28 @@ def exact_fields(
return value
def canonical_json(
value: Any,
*,
error: type[Exception],
label: str,
max_bytes: int,
ensure_ascii: bool,
) -> str:
"""Sorted, compact JSON bounded by ``max_bytes`` of UTF-8."""
def text(value: Any, *, error: type[Exception], label: str, max_bytes: int, strip: bool = True) -> str:
"""Non-blank string bounded by ``max_bytes`` of UTF-8; ``strip=False`` keeps and measures the raw text."""
if not isinstance(value, str):
raise error(f"{label} must be a string")
if not value.strip():
raise error(f"{label} must not be empty")
if strip:
value = value.strip()
if len(value.encode("utf-8")) > max_bytes:
raise error(f"{label} is too large")
return value
def compact_json(value: Any, *, ensure_ascii: bool = True) -> str:
"""Sorted-key, separator-free JSON (the digest/storage canonical form)."""
return json.dumps(value, ensure_ascii=ensure_ascii, sort_keys=True, separators=(",", ":"))
def canonical_json(value: Any, *, error: type[Exception], label: str, max_bytes: int, ensure_ascii: bool) -> str:
"""``compact_json`` bounded by ``max_bytes`` of UTF-8; unserializable input raises ``error``."""
try:
encoded = json.dumps(
value, ensure_ascii=ensure_ascii, sort_keys=True, separators=(",", ":")
)
encoded = compact_json(value, ensure_ascii=ensure_ascii)
except (TypeError, ValueError, RecursionError) as exc:
raise error(f"{label} must be JSON-serializable") from exc
if len(encoded.encode("utf-8")) > max_bytes:
@@ -99,6 +94,11 @@ def utf8_len(*parts: str) -> int:
return len("".join(parts).encode("utf-8"))
def clock(now: float | None) -> float:
"""``now`` as a float, or the current wall clock when ``None``."""
return time.time() if now is None else float(now)
def open_sqlite(path: Path | str, *, timeout: float = 10) -> sqlite3.Connection:
"""Row-factory connection with foreign keys on; no journal or schema work."""
conn = sqlite3.connect(path, timeout=timeout)
@@ -107,10 +107,46 @@ def open_sqlite(path: Path | str, *, timeout: float = 10) -> sqlite3.Connection:
return conn
def connect(
db_path: Path | str, *, db_label: str, ready: Callable[[sqlite3.Connection], bool],
initialize: Callable[[sqlite3.Connection], None], lock_retries: int = 1) -> sqlite3.Connection:
"""Open the shared root store: WAL, foreign keys, then ``initialize`` in one IMMEDIATE txn if not ``ready``.
Multiple profile gateways share this database, so every draft-schema transition
is serialized in SQLite itself: a crash rolls back the whole DDL/data migration and
another process can safely retry it. Only the transient "database is locked" class
from the journal-mode pragma is retried (it may ignore the busy timeout while another
first opener initializes the DB, especially on Windows).
"""
from hermes_state import apply_wal_with_fallback
path = Path(db_path)
path.parent.mkdir(parents=True, exist_ok=True)
conn = sqlite3.connect(path, timeout=10)
conn.row_factory = sqlite3.Row
try:
for attempt in range(lock_retries):
try:
apply_wal_with_fallback(conn, db_label=db_label)
break
except sqlite3.OperationalError as exc:
if str(exc).lower() != "database is locked" or attempt + 1 == lock_retries:
raise
time.sleep(0.01 * (2**attempt))
conn.execute("PRAGMA foreign_keys=ON")
if not ready(conn):
conn.execute("BEGIN IMMEDIATE")
initialize(conn)
conn.commit()
except Exception:
conn.rollback()
conn.close()
raise
return conn
def table_exists(conn: sqlite3.Connection, table: str) -> bool:
row = conn.execute(
"SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (table,)
).fetchone()
row = conn.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (table,)).fetchone()
return row is not None
@@ -120,10 +156,7 @@ def table_columns(conn: sqlite3.Connection, table: str) -> frozenset[str]:
@contextmanager
def transaction(
connect: Callable[[Path | str], sqlite3.Connection],
db_path: Path | str,
*,
immediate: bool,
connect: Callable[[Path | str], sqlite3.Connection], db_path: Path | str, *, immediate: bool
) -> Iterator[sqlite3.Connection]:
"""Open via ``connect``, optionally ``BEGIN IMMEDIATE``, commit on success, always close."""
conn = connect(db_path)
gateway/kanban_watchers.py
+111 -260
View File
@@ -3,19 +3,27 @@
Background loops that subscribe to kanban boards, deliver notifications and
artifacts, and drive the multi-agent dispatcher. They use only ``self`` state,
so they live on a mixin ``GatewayRunner`` inherits. Per-tick work lives in
``kanban_watchers_notifier`` / ``kanban_watchers_dispatcher``.
``kanban_watchers_notifier`` / ``kanban_watchers_dispatcher``; shared plumbing
in ``kanban_watchers_common``.
"""
from __future__ import annotations
import asyncio
import contextlib
import os
import time
from pathlib import Path
from typing import Any, Callable, Optional
from typing import Any, Optional
from gateway.kanban_watchers_common import _to_thread_process_service, logger # noqa: F401 (tests import via origin)
from gateway.kanban_watchers_common import ( # noqa: F401 (tests import via origin)
_acquire_singleton_lock,
_kanban_dispatch_allowed,
_release_singleton_lock,
_resolve_auto_decompose_settings,
_gc_retention_days,
_to_thread_process_service,
logger,
)
from gateway.kanban_watchers_notifier import ( # noqa: F401 (_wake_scope_id: tests import via origin)
_KanbanNotification,
_notifier_collect,
@@ -27,100 +35,16 @@ from gateway.kanban_watchers_dispatcher import (
_resolve_dispatcher_settings,
)
def _resolve_auto_decompose_settings(
load_config: Callable[[], Any],
) -> "tuple[bool, int]":
"""Resolve the live (enabled, per_tick) auto-decompose settings.
Read fresh every dispatcher tick so flipping ``kanban.auto_decompose:
false`` stops runaway fan-out on the next tick without a restart. Fails
safe: a config read error returns ``(False, 3)`` rather than re-enabling
a feature the user turned off. ``per_tick`` is clamped to ``>= 1``.
"""
try:
cfg = load_config()
except Exception:
return False, 3
kcfg = cfg.get("kanban", {}) if isinstance(cfg, dict) else {}
enabled = bool(kcfg.get("auto_decompose", True))
try:
per_tick = int(kcfg.get("auto_decompose_per_tick", 3) or 3)
except (TypeError, ValueError):
per_tick = 3
return enabled, max(per_tick, 1)
def _kanban_dispatch_allowed() -> bool:
"""Return False while the global emergency stop (`hermes pause`) is engaged.
Checked every tick before spawning, so a pause applies on the next tick;
in-flight workers are never touched. Fails open if estop is unimportable.
"""
try:
from agent.estop import check_paused
except ImportError:
return True
return not check_paused("kanban", logger)
def _acquire_singleton_lock(lock_path) -> "tuple[Optional[object], str]":
"""Take the exclusive, non-blocking advisory lock for the sole dispatcher.
Only one gateway machine-wide may run the embedded dispatcher: concurrent
dispatchers double reclaim frequency and claim events, and with
``wal_autocheckpoint=0`` concurrent manual checkpoints can corrupt index
pages. ``dispatch_in_gateway`` is the primary control; this is the backstop.
Returns ``(handle, "held")`` (caller must release via
:func:`_release_singleton_lock`), ``(None, "contended")`` when another
process holds it (caller must NOT dispatch), or ``(None, "unavailable")``
when locking cannot be performed (caller falls back to config control).
"""
try:
from gateway.status import _try_acquire_file_lock # deferred; same package
except ImportError:
return None, "unavailable"
try:
Path(lock_path).parent.mkdir(parents=True, exist_ok=True)
handle = open(str(lock_path), "a+", encoding="utf-8")
except OSError:
return None, "unavailable"
if not _try_acquire_file_lock(handle):
handle.close()
return None, "contended"
return handle, "held"
def _release_singleton_lock(handle) -> None:
"""Release a lock acquired via :func:`_acquire_singleton_lock`."""
if handle is None:
return
try:
from gateway.status import _release_file_lock
_release_file_lock(handle)
except Exception:
pass
with contextlib.suppress(Exception):
handle.close()
def _gc_retention_days() -> int:
"""``kanban.done_sub_retention_days`` (default 30; 0 disables), re-read per sweep; fails safe to 30."""
try:
from hermes_cli.config import load_config as _load_cfg
_kanban_cfg = (_load_cfg() or {}).get("kanban") or {}
return int(_kanban_cfg.get("done_sub_retention_days", 30))
except Exception:
return 30
_IMAGE_EXTS = {".png", ".jpg", ".jpeg", ".gif", ".webp"}
_VIDEO_EXTS = {".mp4", ".mov", ".avi", ".mkv", ".webm", ".3gp"}
_GC_INTERVAL_SECONDS = 3600.0
_HEALTH_WINDOW = 6
class GatewayKanbanWatchersMixin:
"""Kanban watcher / notifier / dispatcher loops for GatewayRunner."""
def _owns_kanban_dispatcher_lock(self) -> bool:
"""Return whether this gateway currently owns the singleton lock."""
return getattr(self, "_kanban_dispatcher_lock_handle", None) is not None
def _release_kanban_dispatcher_lock(self) -> None:
@@ -141,15 +65,12 @@ class GatewayKanbanWatchersMixin:
"""Poll ``kanban_notify_subs`` and deliver terminal events to users.
Per subscription, claims ``task_events`` newer than the stored cursor
(kinds in TERMINAL_KINDS), sends one message per event to
``(platform, chat_id, thread_id)``, then advances the cursor. The
subscription is removed only when the task is ``archived``: ``done``
is reversible (review/continuation), so the cursor — not unsubscribing
— is the dedup mechanism. Earlier unsub-on-terminal silently dropped
users when the dispatcher respawned a crashed task.
All SQLite work runs in a thread; one tick's failure never stops the
next. Iterates every board on disk per tick.
(kinds in TERMINAL_KINDS), sends one message per event, then advances
the cursor. The subscription is removed only when the task is
``archived``: ``done`` is reversible, so the cursor — not unsubscribing
— is the dedup mechanism (unsub-on-terminal dropped users when the
dispatcher respawned a crashed task). All SQLite work runs in a thread;
one tick's failure never stops the next.
"""
from gateway.config import Platform as _Platform
try:
@@ -158,23 +79,18 @@ class GatewayKanbanWatchersMixin:
logger.warning("kanban notifier: kanban_db not importable; notifier disabled")
return
sub_fail_counts: dict[tuple, int] = getattr(
self, "_kanban_sub_fail_counts", {}
)
sub_fail_counts: dict[tuple, int] = getattr(self, "_kanban_sub_fail_counts", {})
self._kanban_sub_fail_counts = sub_fail_counts
notifier_profile = getattr(self, "_kanban_notifier_profile", None)
if not notifier_profile:
notifier_profile = self._active_profile_name()
self._kanban_notifier_profile = notifier_profile
notifier_profile = getattr(self, "_kanban_notifier_profile", None) or self._active_profile_name()
self._kanban_notifier_profile = notifier_profile
# Initial delay so the gateway can finish wiring adapters.
await asyncio.sleep(5)
# Stale done-sub GC: subs survive ``done``, so boards that never
# archive would accumulate rows scanned every tick. One DELETE per
# board, at startup and at most hourly.
_GC_INTERVAL_SECONDS = 3600.0
_gc_next_at = 0.0 # 0 → sweep on the first tick after startup
# board, at startup (0 → first tick) and at most hourly.
_gc_next_at = 0.0
while self._running:
try:
@@ -186,9 +102,7 @@ class GatewayKanbanWatchersMixin:
deliveries = await asyncio.to_thread(
_notifier_collect, self, _kb,
notifier_profile=notifier_profile,
gc_due=_gc_due,
gc_retention_days=_retention,
notifier_profile=notifier_profile, gc_due=_gc_due, gc_retention_days=_retention,
)
for d in deliveries:
await _KanbanNotification(
@@ -204,47 +118,23 @@ class GatewayKanbanWatchersMixin:
conn = _kb.connect(board=board)
try:
getattr(_kb, op)(
conn,
task_id=sub["task_id"],
platform=sub["platform"],
chat_id=sub["chat_id"],
thread_id=sub.get("thread_id") or "",
**extra,
conn, task_id=sub["task_id"], platform=sub["platform"], chat_id=sub["chat_id"],
thread_id=sub.get("thread_id") or "", **extra,
)
finally:
conn.close()
def _kanban_advance(
self, sub: dict, cursor: int, board: Optional[str] = None,
) -> None:
"""Advance a subscription's cursor on the board that owns it."""
def _kanban_advance(self, sub: dict, cursor: int, board: Optional[str] = None) -> None:
self._kanban_sub_op(board, "advance_notify_cursor", sub, new_cursor=cursor)
def _kanban_unsub(self, sub: dict, board: Optional[str] = None) -> None:
self._kanban_sub_op(board, "remove_notify_sub", sub)
def _kanban_rewind(
self,
sub: dict,
claimed_cursor: int,
old_cursor: int,
board: Optional[str] = None,
) -> None:
def _kanban_rewind(self, sub: dict, claimed_cursor: int, old_cursor: int, board: Optional[str] = None) -> None:
"""Undo a claimed notification cursor after send failure."""
self._kanban_sub_op(
board, "rewind_notify_cursor", sub,
claimed_cursor=claimed_cursor, old_cursor=old_cursor,
)
self._kanban_sub_op(board, "rewind_notify_cursor", sub, claimed_cursor=claimed_cursor, old_cursor=old_cursor)
async def _deliver_kanban_artifacts(
self,
*,
adapter,
chat_id: str,
metadata: dict,
event_payload: Optional[dict],
task,
) -> None:
async def _deliver_kanban_artifacts(self, *, adapter, chat_id: str, metadata: dict, event_payload: Optional[dict], task) -> None:
"""Upload artifact files referenced by a completed kanban task.
Sources, in priority order: ``event_payload['artifacts']``,
@@ -252,36 +142,21 @@ class GatewayKanbanWatchersMixin:
deduplicated, missing files are skipped (may be mentioned for
reference only), and upload errors are logged, never raised.
"""
candidates: list[str] = []
seen: set[str] = set()
def _add(path: str) -> None:
if not path:
return
expanded = os.path.expanduser(path)
if expanded in seen or not os.path.isfile(expanded):
return
seen.add(expanded)
candidates.append(expanded)
raw_paths: list[str] = []
if isinstance(event_payload, dict):
raw = event_payload.get("artifacts")
if isinstance(raw, (list, tuple)):
for item in raw:
if isinstance(item, str):
_add(item)
raw_paths += [item for item in raw if isinstance(item, str)]
summary = event_payload.get("summary")
if isinstance(summary, str) and summary:
paths, _ = adapter.extract_local_files(summary)
for p in paths:
_add(p)
raw_paths += adapter.extract_local_files(summary)[0]
if task is not None and getattr(task, "result", None):
paths, _ = adapter.extract_local_files(str(task.result))
for p in paths:
_add(p)
raw_paths += adapter.extract_local_files(str(task.result))[0]
candidates: list[str] = []
for path in raw_paths:
expanded = os.path.expanduser(path) if path else ""
if expanded and expanded not in candidates and os.path.isfile(expanded):
candidates.append(expanded)
if not candidates:
return
@@ -290,41 +165,73 @@ class GatewayKanbanWatchersMixin:
if not candidates:
return
_IMAGE_EXTS = {".png", ".jpg", ".jpeg", ".gif", ".webp"}
_VIDEO_EXTS = {".mp4", ".mov", ".avi", ".mkv", ".webm", ".3gp"}
from urllib.parse import quote as _quote
# Images ride one send_multiple_images call (batch uploads on Signal/Slack).
image_paths = [p for p in candidates if Path(p).suffix.lower() in _IMAGE_EXTS]
other_paths = [p for p in candidates if Path(p).suffix.lower() not in _IMAGE_EXTS]
if image_paths:
try:
batch = [(f"file://{_quote(p)}", "") for p in image_paths]
await adapter.send_multiple_images(
chat_id=chat_id, images=batch, metadata=metadata,
)
await adapter.send_multiple_images(chat_id=chat_id, images=batch, metadata=metadata)
except Exception as exc:
logger.warning(
"kanban notifier: image batch upload failed: %s", exc,
)
logger.warning("kanban notifier: image batch upload failed: %s", exc)
for path in other_paths:
try:
if Path(path).suffix.lower() in _VIDEO_EXTS:
await adapter.send_video(
chat_id=chat_id, video_path=path, metadata=metadata,
)
await adapter.send_video(chat_id=chat_id, video_path=path, metadata=metadata)
else:
await adapter.send_document(
chat_id=chat_id, file_path=path, metadata=metadata,
)
await adapter.send_document(chat_id=chat_id, file_path=path, metadata=metadata)
except Exception as exc:
logger.warning(
"kanban notifier: artifact upload (%s) failed: %s",
path, exc,
)
logger.warning("kanban notifier: artifact upload (%s) failed: %s", path, exc)
def _kanban_dispatcher_boot(self) -> Optional[tuple]:
"""Resolve config, kanban_db and the singleton lock; None when the dispatcher must not run.
Config is read once at boot (restart to apply), except the auto-decompose
toggle which is re-read every tick. The env var is an escape hatch to
disable without editing YAML.
"""
try:
from hermes_cli.config import load_config as _load_config
except Exception:
logger.warning("kanban dispatcher: config loader unavailable; disabled")
return None
env_override = os.environ.get("HERMES_KANBAN_DISPATCH_IN_GATEWAY", "").strip().lower()
if env_override in {"0", "false", "no", "off"}:
logger.info("kanban dispatcher: disabled via HERMES_KANBAN_DISPATCH_IN_GATEWAY env")
return None
try:
cfg = _load_config()
except Exception as exc:
logger.warning("kanban dispatcher: cannot load config (%s); disabled", exc)
return None
kanban_cfg = cfg.get("kanban", {}) if isinstance(cfg, dict) else {}
if not kanban_cfg.get("dispatch_in_gateway", True):
logger.info("kanban dispatcher: disabled via config kanban.dispatch_in_gateway=false")
return None
try:
from hermes_cli import kanban_db as _kb
except Exception:
logger.warning("kanban dispatcher: kanban_db not importable; dispatcher disabled")
return None
# Single-dispatcher backstop (see _acquire_singleton_lock). The lock
# lives at the machine-global kanban root, so it serialises ALL gateways.
self._kanban_dispatcher_lock_handle = None
_lock_path = _kb.kanban_home() / "kanban" / ".dispatcher.lock"
_lock_handle, _lock_state = _acquire_singleton_lock(_lock_path)
if _lock_state == "contended":
logger.info("kanban dispatcher: another gateway already holds the dispatcher "
"lock (%s); this gateway will NOT dispatch.", _lock_path)
return None
if _lock_state == "held":
self._kanban_dispatcher_lock_handle = _lock_handle # hold for process lifetime
logger.info("kanban dispatcher: holding singleton dispatcher lock (%s)", _lock_path)
else:
logger.warning("kanban dispatcher: advisory lock unavailable at %s; proceeding "
"on config control alone.", _lock_path)
return _load_config, _kb, kanban_cfg
async def _kanban_dispatcher_watcher(self) -> None:
"""Embedded kanban dispatcher — one tick every `dispatch_interval_seconds`.
@@ -335,57 +242,10 @@ class GatewayKanbanWatchersMixin:
failure never stops the next. Shutdown: ``self._running`` is checked
between ticks and the in-flight ``to_thread`` returns on its own.
"""
# Config is read once at boot (restart to apply), except the
# auto-decompose toggle which is re-read every tick. The env var is
# an escape hatch to disable without editing YAML.
try:
from hermes_cli.config import load_config as _load_config
except Exception:
logger.warning("kanban dispatcher: config loader unavailable; disabled")
boot = self._kanban_dispatcher_boot()
if boot is None:
return
env_override = os.environ.get("HERMES_KANBAN_DISPATCH_IN_GATEWAY", "").strip().lower()
if env_override in {"0", "false", "no", "off"}:
logger.info("kanban dispatcher: disabled via HERMES_KANBAN_DISPATCH_IN_GATEWAY env")
return
try:
cfg = _load_config()
except Exception as exc:
logger.warning("kanban dispatcher: cannot load config (%s); disabled", exc)
return
kanban_cfg = cfg.get("kanban", {}) if isinstance(cfg, dict) else {}
if not kanban_cfg.get("dispatch_in_gateway", True):
logger.info(
"kanban dispatcher: disabled via config kanban.dispatch_in_gateway=false"
)
return
try:
from hermes_cli import kanban_db as _kb
except Exception:
logger.warning("kanban dispatcher: kanban_db not importable; dispatcher disabled")
return
# Single-dispatcher backstop (see _acquire_singleton_lock). The lock
# lives at the machine-global kanban root, so it serialises ALL gateways.
self._kanban_dispatcher_lock_handle = None
_lock_path = _kb.kanban_home() / "kanban" / ".dispatcher.lock"
_lock_handle, _lock_state = _acquire_singleton_lock(_lock_path)
if _lock_state == "contended":
logger.info(
"kanban dispatcher: another gateway already holds the dispatcher "
"lock (%s); this gateway will NOT dispatch.", _lock_path,
)
return
if _lock_state == "held":
self._kanban_dispatcher_lock_handle = _lock_handle # hold for process lifetime
logger.info("kanban dispatcher: holding singleton dispatcher lock (%s)", _lock_path)
else:
logger.warning(
"kanban dispatcher: advisory lock unavailable at %s; proceeding "
"on config control alone.", _lock_path,
)
_load_config, _kb, kanban_cfg = boot
settings = _resolve_dispatcher_settings(kanban_cfg, _kb)
interval = settings.interval
@@ -395,25 +255,18 @@ class GatewayKanbanWatchersMixin:
# Health telemetry (mirrors `_cmd_daemon`): warn when the ready queue
# is non-empty but spawns are 0 for N consecutive ticks — usually a
# broken PATH, missing venv, or credential loss.
HEALTH_WINDOW = 6
bad_ticks = 0
last_warn_at = 0
dispatcher = _KanbanDispatcher(_kb, settings)
logger.info(
"kanban dispatcher: embedded in gateway (interval=%.1fs)", interval
)
logger.info("kanban dispatcher: embedded in gateway (interval=%.1fs)", interval)
while self._running:
try:
# Reap zombies before per-board work so a board DB failure
# cannot block cleanup of unrelated workers.
pids = await _to_thread_process_service(_kb.reap_worker_zombies)
if pids:
logger.info(
"kanban dispatcher: reaped %d zombie worker(s), pids=%s",
len(pids),
pids,
)
logger.info("kanban dispatcher: reaped %d zombie worker(s), pids=%s", len(pids), pids)
except Exception:
logger.exception("kanban dispatcher: zombie reaper failed")
@@ -430,20 +283,18 @@ class GatewayKanbanWatchersMixin:
await _to_thread_process_service(dispatcher.auto_decompose_tick, _ad_per_tick)
results = await _to_thread_process_service(dispatcher.tick_once)
any_spawned = _log_spawn_results(results)
# Health telemetry (aggregate across boards)
ready_pending = await _to_thread_process_service(dispatcher.ready_nonempty)
bad_ticks = bad_ticks + 1 if ready_pending and not any_spawned else 0
if bad_ticks >= HEALTH_WINDOW:
now = int(time.time())
if now - last_warn_at >= 300:
logger.warning(
"kanban dispatcher stuck: ready queue non-empty for "
"%d consecutive ticks but 0 workers spawned. Check "
"profile health (venv, PATH, credentials) and "
"`hermes kanban list --status ready`.",
bad_ticks,
)
last_warn_at = now
now = int(time.time())
if bad_ticks >= _HEALTH_WINDOW and now - last_warn_at >= 300:
logger.warning(
"kanban dispatcher stuck: ready queue non-empty for "
"%d consecutive ticks but 0 workers spawned. Check "
"profile health (venv, PATH, credentials) and "
"`hermes kanban list --status ready`.",
bad_ticks,
)
last_warn_at = now
except asyncio.CancelledError:
logger.debug("kanban dispatcher: cancelled")
self._release_kanban_dispatcher_lock()
gateway/kanban_watchers_common.py
+111 -2
View File
@@ -1,11 +1,18 @@
"""Plumbing shared by the kanban notifier and dispatcher loops."""
"""Plumbing shared by the kanban notifier and dispatcher loops.
Thread offload, board enumeration, live-config coercers and the singleton
dispatcher lock live here so the notifier, dispatcher and mixin modules read
them from one place.
"""
from __future__ import annotations
import asyncio
import contextlib
import logging
from contextvars import Context
from typing import Any, Callable
from pathlib import Path
from typing import Any, Callable, Optional
# Keep the logger name run.py used so extracted log records are unchanged.
logger = logging.getLogger("gateway.run")
@@ -33,3 +40,105 @@ def _list_boards(kb: Any) -> list:
return kb.list_boards(include_archived=False)
except Exception:
return [kb.read_board_metadata(kb.DEFAULT_BOARD)]
def _board_slugs(kb: Any) -> list:
return [b.get("slug") or kb.DEFAULT_BOARD for b in _list_boards(kb)]
def _positive_int_setting(kanban_cfg: dict, key: str) -> Optional[int]:
"""Parse an optional ``kanban.<key>`` int cap; None when unset or invalid (< 1 is invalid)."""
raw = kanban_cfg.get(key)
if raw is None:
return None
try:
value = int(raw)
except (TypeError, ValueError):
logger.warning("kanban dispatcher: invalid kanban.%s=%r; ignoring", key, raw)
return None
if value < 1:
logger.warning("kanban dispatcher: kanban.%s=%r is below 1; ignoring", key, raw)
return None
logger.info("kanban dispatcher: %s=%d", key, value)
return value
def _resolve_auto_decompose_settings(load_config: Callable[[], Any]) -> "tuple[bool, int]":
"""Live (enabled, per_tick) auto-decompose settings, re-read every dispatcher tick.
Fails safe: a config read error returns ``(False, 3)`` rather than
re-enabling a feature the user turned off. ``per_tick`` is clamped to ``>= 1``.
"""
try:
cfg = load_config()
except Exception:
return False, 3
kcfg = cfg.get("kanban", {}) if isinstance(cfg, dict) else {}
try:
per_tick = int(kcfg.get("auto_decompose_per_tick", 3) or 3)
except (TypeError, ValueError):
per_tick = 3
return bool(kcfg.get("auto_decompose", True)), max(per_tick, 1)
def _gc_retention_days() -> int:
"""``kanban.done_sub_retention_days`` (default 30; 0 disables), re-read per sweep; fails safe to 30."""
try:
from hermes_cli.config import load_config
return int(((load_config() or {}).get("kanban") or {}).get("done_sub_retention_days", 30))
except Exception:
return 30
def _kanban_dispatch_allowed() -> bool:
"""False while the global emergency stop (`hermes pause`) is engaged.
Checked every tick before spawning, so a pause applies on the next tick;
in-flight workers are never touched. Fails open if estop is unimportable.
"""
try:
from agent.estop import check_paused
except ImportError:
return True
return not check_paused("kanban", logger)
def _acquire_singleton_lock(lock_path) -> "tuple[Optional[object], str]":
"""Take the exclusive, non-blocking advisory lock for the sole dispatcher.
Only one gateway machine-wide may run the embedded dispatcher: concurrent
dispatchers double reclaim frequency and claim events, and with
``wal_autocheckpoint=0`` concurrent manual checkpoints can corrupt index
pages. ``dispatch_in_gateway`` is the primary control; this is the backstop.
Returns ``(handle, "held")`` (release via :func:`_release_singleton_lock`),
``(None, "contended")`` when another process holds it (caller must NOT
dispatch), or ``(None, "unavailable")`` when locking cannot be performed
(caller falls back to config control).
"""
try:
from gateway.status import _try_acquire_file_lock # deferred; same package
except ImportError:
return None, "unavailable"
try:
Path(lock_path).parent.mkdir(parents=True, exist_ok=True)
handle = open(str(lock_path), "a+", encoding="utf-8")
except OSError:
return None, "unavailable"
if not _try_acquire_file_lock(handle):
handle.close()
return None, "contended"
return handle, "held"
def _release_singleton_lock(handle) -> None:
"""Release a lock acquired via :func:`_acquire_singleton_lock`."""
if handle is None:
return
with contextlib.suppress(Exception):
from gateway.status import _release_file_lock
_release_file_lock(handle)
with contextlib.suppress(Exception):
handle.close()
gateway/kanban_watchers_dispatcher.py
+42 -119
View File
@@ -11,10 +11,10 @@ import contextlib
import os
import sqlite3
import time
from dataclasses import dataclass
from dataclasses import asdict, dataclass
from typing import Any, Optional
from gateway.kanban_watchers_common import _list_boards, logger
from gateway.kanban_watchers_common import _board_slugs, _positive_int_setting, logger
_CORRUPT_DB_MARKERS = ("file is not a database", "database disk image is malformed")
@@ -33,32 +33,13 @@ class _DispatcherSettings:
max_in_progress_per_profile: Optional[int]
def _positive_int_setting(kanban_cfg: dict, key: str) -> Optional[int]:
"""Parse an optional ``kanban.<key>`` int cap; None when unset or invalid (< 1 is invalid)."""
raw = kanban_cfg.get(key)
if raw is None:
return None
try:
value = int(raw)
except (TypeError, ValueError):
logger.warning("kanban dispatcher: invalid kanban.%s=%r; ignoring", key, raw)
return None
if value < 1:
logger.warning("kanban dispatcher: kanban.%s=%r is below 1; ignoring", key, raw)
return None
logger.info("kanban dispatcher: %s=%d", key, value)
return value
def _resolve_dispatcher_settings(kanban_cfg: dict, kb: Any) -> _DispatcherSettings:
"""Parse and log the dispatcher settings in their established order."""
try:
interval = float(kanban_cfg.get("dispatch_interval_seconds", 60) or 60)
except (ValueError, TypeError):
logger.warning(
"kanban dispatcher: invalid dispatch_interval_seconds=%r, using default 60",
kanban_cfg.get("dispatch_interval_seconds"),
)
logger.warning("kanban dispatcher: invalid dispatch_interval_seconds=%r, using default 60",
kanban_cfg.get("dispatch_interval_seconds"))
interval = 60.0
interval = max(interval, 1.0) # sanity floor — tighter than this is a footgun
@@ -66,10 +47,9 @@ def _resolve_dispatcher_settings(kanban_cfg: dict, kb: Any) -> _DispatcherSettin
if max_spawn is not None:
logger.info("kanban dispatcher: max_spawn=%s", max_spawn)
# Cap simultaneously running tasks so slow workers don't pile up and
# time out. Explicit config wins; otherwise a memory-derived default
# (unbounded fan-out swap-thrashes small hosts), or None where total
# memory can't be read.
# Cap simultaneously running tasks so slow workers don't pile up and time
# out. Explicit config wins; otherwise a memory-derived default (unbounded
# fan-out swap-thrashes small hosts), or None where total memory can't be read.
max_in_progress = _positive_int_setting(kanban_cfg, "max_in_progress")
effective_max_in_progress = kb.resolve_max_in_progress(max_in_progress)
if max_in_progress is None and effective_max_in_progress is not None:
@@ -84,18 +64,12 @@ def _resolve_dispatcher_settings(kanban_cfg: dict, kb: Any) -> _DispatcherSettin
try:
failure_limit = int(raw_failure_limit)
except (TypeError, ValueError):
logger.warning(
"kanban dispatcher: invalid kanban.failure_limit=%r; using default %d",
raw_failure_limit,
kb.DEFAULT_FAILURE_LIMIT,
)
logger.warning("kanban dispatcher: invalid kanban.failure_limit=%r; using default %d",
raw_failure_limit, kb.DEFAULT_FAILURE_LIMIT)
failure_limit = kb.DEFAULT_FAILURE_LIMIT
if failure_limit < 1:
logger.warning(
"kanban dispatcher: kanban.failure_limit=%r is below 1; using default %d",
raw_failure_limit,
kb.DEFAULT_FAILURE_LIMIT,
)
logger.warning("kanban dispatcher: kanban.failure_limit=%r is below 1; using default %d",
raw_failure_limit, kb.DEFAULT_FAILURE_LIMIT)
failure_limit = kb.DEFAULT_FAILURE_LIMIT
# 0 disables stale detection.
@@ -103,22 +77,16 @@ def _resolve_dispatcher_settings(kanban_cfg: dict, kb: Any) -> _DispatcherSettin
try:
stale_timeout_seconds = int(raw_stale or 0)
except (TypeError, ValueError):
logger.warning(
"kanban dispatcher: invalid kanban.dispatch_stale_timeout_seconds=%r; "
"disabling stale detection",
raw_stale,
)
logger.warning("kanban dispatcher: invalid kanban.dispatch_stale_timeout_seconds=%r; "
"disabling stale detection", raw_stale)
stale_timeout_seconds = 0
# Fallback profile for tasks created without an assignee (e.g. via the
# dashboard). Empty (the schema default) keeps skipping them.
default_assignee = (kanban_cfg.get("default_assignee") or "").strip() or None
if default_assignee:
logger.info(
"kanban dispatcher: default_assignee=%r (unassigned ready tasks "
"will route to this profile)",
default_assignee,
)
logger.info("kanban dispatcher: default_assignee=%r (unassigned ready tasks "
"will route to this profile)", default_assignee)
return _DispatcherSettings(
interval=interval,
@@ -150,12 +118,10 @@ class _KanbanDispatcher:
def __init__(self, kb: Any, settings: _DispatcherSettings) -> None:
self.kb = kb
self.settings = settings
self.disabled_corrupt_boards: dict[
str, tuple[tuple[str, int | None, int | None], float]
] = {}
self.disabled_corrupt_boards: dict[str, tuple[tuple[str, int | None, int | None], float]] = {}
def _board_slugs(self) -> list:
return [b.get("slug") or self.kb.DEFAULT_BOARD for b in _list_boards(self.kb)]
return _board_slugs(self.kb)
def board_db_fingerprint(self, slug: str) -> tuple[str, int | None, int | None]:
path = self.kb.kanban_db_path(slug)
@@ -173,10 +139,7 @@ class _KanbanDispatcher:
corrupt_guard_error = getattr(self.kb, "KanbanDbCorruptError", None)
if corrupt_guard_error is not None and isinstance(exc, corrupt_guard_error):
return True
if not isinstance(exc, sqlite3.DatabaseError):
return False
msg = str(exc).lower()
return any(marker in msg for marker in _CORRUPT_DB_MARKERS)
return isinstance(exc, sqlite3.DatabaseError) and any(m in str(exc).lower() for m in _CORRUPT_DB_MARKERS)
def _quarantine_lifted(self, slug: str, fingerprint: tuple) -> bool:
"""Return False while *slug* stays quarantined; lift (and log) otherwise."""
@@ -188,17 +151,10 @@ class _KanbanDispatcher:
if disabled_fingerprint == fingerprint and age < self.CORRUPT_BOARD_RETRY_AFTER_SECONDS:
return False
if disabled_fingerprint == fingerprint:
logger.info(
"kanban dispatcher: board %s database fingerprint unchanged "
"after %.0fs quarantine; retrying dispatch",
slug,
age,
)
logger.info("kanban dispatcher: board %s database fingerprint unchanged "
"after %.0fs quarantine; retrying dispatch", slug, age)
else:
logger.info(
"kanban dispatcher: board %s database changed; retrying dispatch",
slug,
)
logger.info("kanban dispatcher: board %s database changed; retrying dispatch", slug)
self.disabled_corrupt_boards.pop(slug, None)
return True
@@ -212,22 +168,12 @@ class _KanbanDispatcher:
fingerprint = self.board_db_fingerprint(slug)
if not self._quarantine_lifted(slug, fingerprint):
return None
s = self.settings
kwargs = {k: v for k, v in asdict(self.settings).items() if k != "interval"}
try:
# No explicit init_db(): connect() runs the migration once per
# process (see the matching note in the notifier collector).
conn = self.kb.connect(board=slug)
return self.kb.dispatch_once(
conn,
board=slug,
max_spawn=s.max_spawn,
max_in_progress=s.max_in_progress,
failure_limit=s.failure_limit,
stale_timeout_seconds=s.stale_timeout_seconds,
default_assignee=s.default_assignee,
max_in_progress_per_profile=s.max_in_progress_per_profile,
reconcile_orphans=s.reconcile_orphans,
)
return self.kb.dispatch_once(conn, board=slug, **kwargs)
except Exception as exc:
if self.is_corrupt_board_db_error(exc):
self.disabled_corrupt_boards[slug] = (fingerprint, time.monotonic())
@@ -237,8 +183,7 @@ class _KanbanDispatcher:
"the file changes, the gateway restarts, or the "
"quarantine timer expires. Move or restore the file, "
"then run `hermes kanban init` if you need a fresh board.",
slug,
fingerprint[0],
slug, fingerprint[0],
)
return None
logger.exception("kanban dispatcher: tick failed on board %s", slug)
@@ -253,14 +198,13 @@ class _KanbanDispatcher:
return [(slug, self.tick_once_for_board(slug)) for slug in self._board_slugs()]
def ready_nonempty(self) -> bool:
"""Is there a ready+assigned+unclaimed task on ANY board that the
dispatcher would actually spawn for?
"""Is there a ready+assigned+unclaimed task on ANY board the dispatcher would spawn for?
Control-plane lanes (e.g. ``orion-cc``) are pulled by terminals
via ``claim_task`` and never spawnable — a queue full of those is
"correctly idle", not "stuck". The review column is probed only
when review dispatch is on (same gate as the dispatcher): a task
waiting for a human reviewer is idle, not stuck.
Control-plane lanes (e.g. ``orion-cc``) are pulled by terminals via
``claim_task`` and never spawnable — a queue full of those is
"correctly idle", not "stuck". The review column is probed only when
review dispatch is on (same gate as the dispatcher): a task waiting
for a human reviewer is idle, not stuck.
"""
kb = self.kb
_review_probe = kb.review_dispatch_enabled()
@@ -268,9 +212,7 @@ class _KanbanDispatcher:
conn = None
try:
conn = kb.connect(board=slug)
if kb.has_spawnable_ready(conn):
return True
if _review_probe and kb.has_spawnable_review(conn):
if kb.has_spawnable_ready(conn) or (_review_probe and kb.has_spawnable_review(conn)):
return True
except Exception:
continue
@@ -281,17 +223,15 @@ class _KanbanDispatcher:
return False
def auto_decompose_tick(self, auto_decompose_per_tick: int) -> int:
"""Auto-decompose up to N triage tasks across all boards into
ready workgraphs before dispatch fans out; the per-tick cap keeps
a bulk triage load from burst-spending the aux LLM. Returns the
number decomposed/specified.
"""Auto-decompose up to N triage tasks across all boards into ready workgraphs.
Runs before dispatch fans out; the per-tick cap keeps a bulk triage
load from burst-spending the aux LLM. Returns the number decomposed.
"""
try:
from hermes_cli import kanban_decompose as _decomp
except Exception as exc: # pragma: no cover
logger.warning(
"kanban auto-decompose: import failed (%s); skipping", exc,
)
logger.warning("kanban auto-decompose: import failed (%s); skipping", exc)
return 0
attempted = 0
successes = 0
@@ -306,10 +246,7 @@ class _KanbanDispatcher:
try:
triage_ids = _decomp.list_triage_ids()
except Exception as exc:
logger.debug(
"kanban auto-decompose: list_triage_ids failed on board %s (%s)",
slug, exc,
)
logger.debug("kanban auto-decompose: list_triage_ids failed on board %s (%s)", slug, exc)
triage_ids = []
for tid in triage_ids:
if attempted >= auto_decompose_per_tick:
@@ -329,28 +266,16 @@ class _KanbanDispatcher:
try:
outcome = _decomp.decompose_task(tid, author="auto-decomposer")
except Exception:
logger.exception(
"kanban auto-decompose: decompose_task crashed on %s",
tid,
)
logger.exception("kanban auto-decompose: decompose_task crashed on %s", tid)
return 0
if not outcome.ok:
# Common no-op reasons (no aux client) must not spam logs every tick.
logger.debug(
"kanban auto-decompose [%s]: %s skipped: %s",
slug, tid, outcome.reason,
)
logger.debug("kanban auto-decompose [%s]: %s skipped: %s", slug, tid, outcome.reason)
return 0
if outcome.fanout and outcome.child_ids:
logger.info(
"kanban auto-decompose [%s]: %s → %d children",
slug, tid, len(outcome.child_ids),
)
logger.info("kanban auto-decompose [%s]: %s → %d children", slug, tid, len(outcome.child_ids))
else:
logger.info(
"kanban auto-decompose [%s]: %s → single task (no fanout)",
slug, tid,
)
logger.info("kanban auto-decompose [%s]: %s → single task (no fanout)", slug, tid)
return 1
@@ -364,9 +289,7 @@ def _log_spawn_results(results: Optional[list]) -> bool:
logger.info(
"kanban dispatcher [%s]: spawned=%d reclaimed=%d "
"crashed=%d timed_out=%d promoted=%d auto_blocked=%d",
slug,
len(res.spawned),
res.reclaimed,
slug, len(res.spawned), res.reclaimed,
len(res.crashed) if hasattr(res.crashed, "__len__") else 0,
len(res.timed_out) if hasattr(res.timed_out, "__len__") else 0,
res.promoted,
gateway/kanban_watchers_notifier.py
+228 -390
View File
@@ -21,34 +21,21 @@ from gateway.kanban_watchers_common import _list_boards, _to_thread_process_serv
TERMINAL_KINDS = ("completed", "blocked", "gave_up", "crashed", "timed_out", "status", "archived", "unblocked", "block_loop_detected", "review_requested", "changes_requested")
# Kinds that hand a decision back to the origin, which must take a turn.
# status/archived/unblocked are bookkeeping.
_WAKE_KINDS = (
"completed", "gave_up", "crashed", "timed_out",
"blocked", "review_requested", "changes_requested",
"block_loop_detected",
)
# Consecutive send failures (adapter raised OR reported
# SendResult(success=False)) before a sub is dropped as a dead chat.
# 12 ≈ 60s at the 5s cadence: a transient API outage must not
# permanently unsubscribe a live review-gate channel.
_WAKE_KINDS = ("completed", "gave_up", "crashed", "timed_out", "blocked", "review_requested", "changes_requested", "block_loop_detected")
# Consecutive send failures (adapter raised OR reported SendResult(success=False))
# before a sub is dropped as a dead chat. 12 ≈ 60s at the 5s cadence: a transient
# API outage must not permanently unsubscribe a live review-gate channel.
MAX_SEND_FAILURES = 12
_LOCAL_PATH_RE = re.compile(
r"(?<![\w:/])(?:/(?:Users|home|private|tmp|var|etc|workspace)/[^\s,;]+|"
r"[A-Za-z]:\\[^\s,;]+)"
)
_LOCAL_PATH_RE = re.compile(r"(?<![\w:/])(?:/(?:Users|home|private|tmp|var|etc|workspace)/[^\s,;]+|" r"[A-Za-z]:\\[^\s,;]+)")
def _safe_review_reason(value: Any, limit: int = 160) -> str:
"""Return a mobile-friendly review reason safe for external delivery."""
from agent.redact import redact_sensitive_text
reason = redact_sensitive_text(
"" if value is None else str(value),
force=True,
redact_url_credentials=True,
)
reason = _LOCAL_PATH_RE.sub("[local path]", reason)
reason = " ".join(reason.split())
reason = redact_sensitive_text("" if value is None else str(value), force=True, redact_url_credentials=True)
reason = " ".join(_LOCAL_PATH_RE.sub("[local path]", reason).split())
if len(reason) > limit:
reason = reason[: limit - 1].rstrip() + "…"
return reason
@@ -70,21 +57,15 @@ def _wake_scope_id(adapter: Any, sub: dict) -> Optional[str]:
if value:
return str(value)
resolver = getattr(adapter, "scope_id_for_chat", None)
if callable(resolver):
try:
resolved = resolver(str(sub.get("chat_id") or ""))
except Exception as exc:
# An adapter-side lookup failure yields no scope, never an error.
logger.debug(
"kanban notifier: scope lookup failed for chat %s: %s",
sub.get("chat_id"),
exc,
exc_info=True,
)
return None
if resolved:
return str(resolved)
return None
if not callable(resolver):
return None
try:
resolved = resolver(str(sub.get("chat_id") or ""))
except Exception as exc:
# An adapter-side lookup failure yields no scope, never an error.
logger.debug("kanban notifier: scope lookup failed for chat %s: %s", sub.get("chat_id"), exc, exc_info=True)
return None
return str(resolved) if resolved else None
def _platform_names(mapping: Any) -> set[str]:
@@ -92,260 +73,177 @@ def _platform_names(mapping: Any) -> set[str]:
return {getattr(platform, "value", str(platform)).lower() for platform in mapping}
# ---------------------------------------------------------------------------
# Collection (runs in a worker thread)
# ---------------------------------------------------------------------------
# --- Collection (runs in a worker thread) ---
def _notifier_collect(
runner: Any,
kb: Any,
*,
notifier_profile: Optional[str],
gc_due: bool,
gc_retention_days: int,
) -> list[dict]:
class _Collector:
"""One tick's claim state: which profiles/platforms this gateway serves and the GC gate."""
def __init__(self, runner: Any, kb: Any, *, notifier_profile: Optional[str], gc_due: bool, gc_retention_days: int) -> None:
self.kb = kb
self.notifier_profile = notifier_profile
self.gc_due = gc_due
self.gc_retention_days = gc_retention_days
self.deliveries: list[dict] = []
self.include_unowned = runner._owns_kanban_dispatcher_lock()
self.profile_adapters = getattr(runner, "_profile_adapters", {})
self.notifier_profiles = {notifier_profile}
self.notifier_profiles.update(str(p).strip() for p in self.profile_adapters if str(p).strip())
# Include every platform any secondary profile has live. This is only a
# coarse pre-filter; the precise per-profile check (_authorization_adapter,
# no default fallback) runs at delivery and rewinds the claim if it
# resolves to None. An unclaimed event never retries, so dropping a
# secondary-profile sub here would lose it.
self.active_platforms = _platform_names(runner.adapters).union(
*(_platform_names(m) for m in self.profile_adapters.values()))
def collect(self) -> list[dict]:
if not self.active_platforms:
logger.debug("kanban notifier: no connected adapters; skipping tick")
return self.deliveries
# Poll each resolved DB path once: several slugs can map to one DB when
# HERMES_KANBAN_DB pins the board path.
kb = self.kb
seen_db_paths: set[str] = set()
for board_meta in _list_boards(kb):
slug = board_meta.get("slug") or kb.DEFAULT_BOARD
db_path = board_meta.get("db_path")
try:
resolved_db_path = str(Path(db_path).expanduser().resolve()) if db_path else str(kb.kanban_db_path(slug).resolve())
except Exception:
resolved_db_path = f"slug:{slug}"
if resolved_db_path in seen_db_paths:
logger.debug("kanban notifier: skipping duplicate board slug %s for DB %s", slug, resolved_db_path)
continue
seen_db_paths.add(resolved_db_path)
self.collect_board(slug)
return self.deliveries
def _board_has_subs(self, slug: str) -> bool:
"""Cheap read-only probe before the writable connect() (schema init, WAL
sidecars, checkpoints); a probe failure falls back to the writable open."""
try:
count = self.kb.count_notify_subs(
board=slug, notifier_profiles=self.notifier_profiles, include_unowned=self.include_unowned)
except Exception as exc:
logger.debug("kanban notifier: read-only subscription probe failed "
"for board %s (%s); falling back to writable open", slug, exc)
return True
if count == 0:
logger.debug("kanban notifier: board %s has no subscriptions owned by %s; skipping open",
slug, sorted(self.notifier_profiles))
return count != 0
def _gc_stale_subs(self, conn: Any, slug: str) -> None:
"""Best-effort stale-sub sweep: a failed sweep never blocks delivery; the next hourly gate retries."""
try:
_purged = self.kb.purge_stale_done_notify_subs(conn, max_age_days=self.gc_retention_days)
if _purged:
logger.info("kanban notifier: purged %d stale done/blocked-task subscription(s) on board %s (retention %dd)",
_purged, slug, self.gc_retention_days)
except Exception as _gc_exc:
logger.debug("kanban notifier: stale-sub GC failed for board %s: %s", slug, _gc_exc)
def _claim_for_sub(self, conn: Any, slug: str, sub: dict) -> Optional[dict]:
"""Claim one subscription's unseen events; None when skipped or nothing new."""
owner_profile = sub.get("notifier_profile") or None
if owner_profile and owner_profile != self.notifier_profile and not self.profile_adapters.get(owner_profile):
logger.debug("kanban notifier: subscription for %s owned by profile %s; current profile %s has no adapter for it, skipping",
sub.get("task_id"), owner_profile, self.notifier_profile)
return None
platform = (sub.get("platform") or "").lower()
if platform not in self.active_platforms:
logger.debug("kanban notifier: subscription for %s on %s skipped; adapter not connected",
sub.get("task_id"), platform or "<missing>")
return None
old_cursor, cursor, events = self.kb.claim_unseen_events_for_sub(
conn, task_id=sub["task_id"], platform=sub["platform"], chat_id=sub["chat_id"],
thread_id=sub.get("thread_id") or "", kinds=TERMINAL_KINDS,
)
if not events:
return None
task = self.kb.get_task(conn, sub["task_id"])
logger.debug("kanban notifier: claimed %d event(s) for %s on board %s cursor %s→%s",
len(events), sub["task_id"], slug, old_cursor, cursor)
return {"sub": sub, "old_cursor": old_cursor, "cursor": cursor, "events": events, "task": task, "board": slug}
def collect_board(self, slug: str) -> None:
"""Claim events on one board, appending delivery dicts to ``deliveries``."""
if not self._board_has_subs(slug):
return
kb = self.kb
try:
conn = kb.connect(board=slug)
except Exception as exc:
logger.debug("kanban notifier: cannot open board %s: %s", slug, exc)
return
try:
if self.gc_due:
self._gc_stale_subs(conn, slug)
# No explicit init_db(): connect() already runs the migration once per
# process, and init_db() would re-run it on a second connection racing
# the first.
subs = kb.list_notify_subs(conn, notifier_profiles=self.notifier_profiles, include_unowned=self.include_unowned)
if not subs:
logger.debug("kanban notifier: board %s has no subscriptions", slug)
for sub in subs:
try:
claimed = self._claim_for_sub(conn, slug, sub)
if claimed is not None:
self.deliveries.append(claimed)
except Exception as sub_exc:
# One bad subscription must not block the rest of the tick.
logger.warning("kanban notifier: subscription for %s on board %s failed: %s",
sub.get("task_id"), slug, sub_exc)
finally:
conn.close()
def _notifier_collect(runner: Any, kb: Any, *, notifier_profile: Optional[str], gc_due: bool, gc_retention_days: int) -> list[dict]:
"""Claim unseen terminal events for every owned subscription on every board.
Each gateway polls only subscriptions owned by profiles whose adapters it
hosts; legacy rows without a profile stamp are visible only to the process
holding the singleton dispatcher lock.
"""
deliveries: list[dict] = []
include_unowned = runner._owns_kanban_dispatcher_lock()
profile_adapters = getattr(runner, "_profile_adapters", {})
notifier_profiles = {notifier_profile}
notifier_profiles.update(
str(profile).strip() for profile in profile_adapters if str(profile).strip()
)
active_platforms = _platform_names(runner.adapters)
# Include every platform any secondary profile has live. This is only a
# coarse pre-filter; the precise per-profile check (_authorization_adapter,
# no default fallback) runs at delivery and rewinds the claim if it
# resolves to None. An unclaimed event never retries, so dropping a
# secondary-profile sub here would lose it.
for _profile_adapter_map in profile_adapters.values():
active_platforms.update(_platform_names(_profile_adapter_map))
if not active_platforms:
logger.debug("kanban notifier: no connected adapters; skipping tick")
return deliveries
# Poll each resolved DB path once: several slugs can map to one DB when
# HERMES_KANBAN_DB pins the board path.
seen_db_paths: set[str] = set()
for board_meta in _list_boards(kb):
slug = board_meta.get("slug") or kb.DEFAULT_BOARD
db_path = board_meta.get("db_path")
try:
resolved_db_path = str(Path(db_path).expanduser().resolve()) if db_path else str(kb.kanban_db_path(slug).resolve())
except Exception:
resolved_db_path = f"slug:{slug}"
if resolved_db_path in seen_db_paths:
logger.debug(
"kanban notifier: skipping duplicate board slug %s for DB %s",
slug, resolved_db_path,
)
continue
seen_db_paths.add(resolved_db_path)
_notifier_collect_board(
kb, slug, deliveries,
notifier_profile=notifier_profile,
notifier_profiles=notifier_profiles,
include_unowned=include_unowned,
profile_adapters=profile_adapters,
active_platforms=active_platforms,
gc_due=gc_due,
gc_retention_days=gc_retention_days,
)
return deliveries
return _Collector(
runner, kb, notifier_profile=notifier_profile, gc_due=gc_due, gc_retention_days=gc_retention_days,
).collect()
def _notifier_collect_board(
kb: Any,
slug: str,
deliveries: list[dict],
*,
notifier_profile: Optional[str],
notifier_profiles: set,
include_unowned: bool,
profile_adapters: dict,
active_platforms: set[str],
gc_due: bool,
gc_retention_days: int,
) -> None:
"""Claim events on one board, appending delivery dicts to *deliveries*."""
# Cheap read-only probe before the writable connect() (schema init, WAL
# sidecars, checkpoints) — a board with no subscriptions has nothing to notify.
try:
if kb.count_notify_subs(
board=slug,
notifier_profiles=notifier_profiles,
include_unowned=include_unowned,
) == 0:
logger.debug(
"kanban notifier: board %s has no subscriptions owned by %s; skipping open",
slug, sorted(notifier_profiles),
)
return
except Exception as exc:
logger.debug(
"kanban notifier: read-only subscription probe failed "
"for board %s (%s); falling back to writable open",
slug, exc,
)
try:
conn = kb.connect(board=slug)
except Exception as exc:
logger.debug("kanban notifier: cannot open board %s: %s", slug, exc)
return
try:
if gc_due:
# Best-effort: a failed sweep never blocks delivery; the next
# hourly gate retries.
try:
_purged = kb.purge_stale_done_notify_subs(conn, max_age_days=gc_retention_days)
if _purged:
logger.info(
"kanban notifier: purged %d stale done/blocked-task subscription(s) on board %s (retention %dd)",
_purged, slug, gc_retention_days,
)
except Exception as _gc_exc:
logger.debug(
"kanban notifier: stale-sub GC failed for board %s: %s",
slug, _gc_exc,
)
# No explicit init_db(): connect() already runs the migration once per
# process, and init_db() would re-run it on a second connection racing
# the first.
subs = kb.list_notify_subs(
conn,
notifier_profiles=notifier_profiles,
include_unowned=include_unowned,
)
if not subs:
logger.debug("kanban notifier: board %s has no subscriptions", slug)
for sub in subs:
try:
owner_profile = sub.get("notifier_profile") or None
if owner_profile and owner_profile != notifier_profile and not profile_adapters.get(owner_profile):
logger.debug(
"kanban notifier: subscription for %s owned by profile %s; current profile %s has no adapter for it, skipping",
sub.get("task_id"), owner_profile, notifier_profile,
)
continue
platform = (sub.get("platform") or "").lower()
if platform not in active_platforms:
logger.debug(
"kanban notifier: subscription for %s on %s skipped; adapter not connected",
sub.get("task_id"), platform or "<missing>",
)
continue
old_cursor, cursor, events = kb.claim_unseen_events_for_sub(
conn,
task_id=sub["task_id"],
platform=sub["platform"],
chat_id=sub["chat_id"],
thread_id=sub.get("thread_id") or "",
kinds=TERMINAL_KINDS,
)
if not events:
continue
task = kb.get_task(conn, sub["task_id"])
logger.debug(
"kanban notifier: claimed %d event(s) for %s on board %s cursor %s→%s",
len(events), sub["task_id"], slug, old_cursor, cursor,
)
deliveries.append({
"sub": sub,
"old_cursor": old_cursor,
"cursor": cursor,
"events": events,
"task": task,
"board": slug,
})
except Exception as sub_exc:
# One bad subscription must not block the rest of the tick.
logger.warning(
"kanban notifier: subscription for %s on board %s failed: %s",
sub.get("task_id"), slug, sub_exc,
)
finally:
conn.close()
# ---------------------------------------------------------------------------
# Per-event message formatting: kind -> (msg, wake_handoff, wake_review_detail)
# ---------------------------------------------------------------------------
# ``None`` for handoff / review_detail leaves the accumulated wake value
# untouched. ``_payload(ev, key)`` is the shared "payload present and truthy" read.
# --- Per-event message formatting: kind -> (msg, wake_handoff, wake_review_detail) ---
# ``None`` for handoff / review_detail leaves the accumulated wake value untouched.
def _payload(ev: Any, key: str) -> Any:
"""Shared "payload present and truthy" read."""
return ev.payload.get(key) if ev.payload and ev.payload.get(key) else None
def _clip(ev: Any, key: str, fmt: str, limit: int) -> str:
"""``fmt`` applied to the truncated payload value, or ``""`` when absent."""
value = _payload(ev, key)
return fmt.format(str(value)[:limit]) if value else ""
_NL = "\n{}"
def _first_line(text: str, limit: int) -> str:
lines = text.strip().splitlines()
return lines[0][:limit] if lines else text[:limit]
def _fmt_completed(ev, n) -> tuple:
# Prefer the run summary from the event payload; fall back to
# task.result for legacy rows.
handoff = ""
# Prefer the run summary from the event payload; fall back to task.result for legacy rows.
wake_handoff = None
payload_summary = _payload(ev, "summary")
if payload_summary:
wake_handoff = _first_line(str(payload_summary), 200)
handoff = f"\n{wake_handoff}"
elif n.task and n.task.result:
wake_handoff = _first_line(n.task.result, 160)
handoff = f"\n{wake_handoff}"
msg = (
f"✔ {n.board_tag}{n.tag}Kanban {n.task_id} done"
f" — {n.title}{handoff}"
)
return msg, wake_handoff, None
def _fmt_blocked(ev, n) -> tuple:
reason = _payload(ev, "reason")
reason = f": {str(reason)[:160]}" if reason else ""
return f"⏸ {n.board_tag}{n.tag}Kanban {n.task_id} blocked{reason}", None, None
def _fmt_gave_up(ev, n) -> tuple:
err = _payload(ev, "error")
err = f"\n{str(err)[:200]}" if err else ""
msg = (
f"✖ {n.board_tag}{n.tag}Kanban {n.task_id} gave up "
f"after repeated spawn failures{err}"
)
return msg, None, None
def _fmt_crashed(ev, n) -> tuple:
msg = (
f"✖ {n.board_tag}{n.tag}Kanban {n.task_id} worker crashed "
f"(pid gone); dispatcher will retry"
)
return msg, None, None
def _fmt_timed_out(ev, n) -> tuple:
limit = _payload(ev, "limit_seconds")
limit = int(limit) if limit else 0
msg = (
f"⏱ {n.board_tag}{n.tag}Kanban {n.task_id} timed out "
f"(max_runtime={limit}s); will retry"
)
return msg, None, None
def _fmt_status(ev, n) -> tuple:
new_status = _payload(ev, "status")
new_status = str(new_status) if new_status else ""
return f"🔄 {n.board_tag}{n.tag}Kanban {n.task_id} → {new_status}", None, None
handoff = f"\n{wake_handoff}" if wake_handoff is not None else ""
return f"✔ {n.head} done — {n.title}{handoff}", wake_handoff, None
def _fmt_review_requested(ev, n) -> tuple:
@@ -358,11 +256,7 @@ def _fmt_review_requested(ev, n) -> tuple:
summary = str(summary)
handoff = f"\n{summary[:200]}"
wake_handoff = _first_line(summary, 200)
msg = (
f"👀 {n.board_tag}{n.tag}Kanban {n.task_id} ready for review"
f" — {n.title}{handoff}"
)
return msg, wake_handoff, None
return f"👀 {n.head} ready for review — {n.title}{handoff}", wake_handoff, None
def _fmt_changes_requested(ev, n) -> tuple:
@@ -371,51 +265,40 @@ def _fmt_changes_requested(ev, n) -> tuple:
reviewer = _safe_review_reason(payload.get("reviewer"), 48)
implementer = _safe_review_reason(payload.get("implementer"), 48)
reason_text = reason or "reviewer feedback requires changes"
provenance = ""
if reviewer:
provenance += f" — reviewer @{reviewer}"
provenance = f" — reviewer @{reviewer}" if reviewer else ""
if implementer:
provenance += f" → implementer @{implementer}"
msg = (
f"🛑 {n.board_tag}Kanban {n.task_id} review requested "
f"changes/BLOCK: {reason_text}{provenance}"
)
msg = f"🛑 {n.board_tag}Kanban {n.task_id} review requested changes/BLOCK: {reason_text}{provenance}"
return msg, None, reason_text
def _fmt_block_loop_detected(ev, n) -> tuple:
# Re-blocked for the same cause past the limit and routed to `triage`
# for a human. It emits no blocked/status event, so ping loudly here.
reason = _payload(ev, "reason")
reason = f": {str(reason)[:160]}" if reason else ""
recurrences = ev.payload.get("recurrences") if ev.payload else None
rc = f" (blocked {recurrences}x for the same cause)" if recurrences else ""
msg = (
f"🛑 {n.board_tag}{n.tag}Kanban {n.task_id} routed to TRIAGE"
f" — needs a human decision{rc}{reason}"
)
return msg, None, None
# archived / unblocked are claimed (so the cursor advances past them) but
# intentionally silent (no formatter), and excluded from _WAKE_KINDS so they
# never wake the creator.
_EVENT_FORMATTERS: dict[str, Callable[[Any, "_KanbanNotification"], tuple]] = {
"completed": _fmt_completed,
"blocked": _fmt_blocked,
"gave_up": _fmt_gave_up,
"crashed": _fmt_crashed,
"timed_out": _fmt_timed_out,
"status": _fmt_status,
"blocked": lambda ev, n: (f"⏸ {n.head} blocked{_clip(ev, 'reason', ': {}', 160)}", None, None),
"gave_up": lambda ev, n: (
f"✖ {n.head} gave up after repeated spawn failures{_clip(ev, 'error', _NL, 200)}", None, None,
),
"crashed": lambda ev, n: (f"✖ {n.head} worker crashed (pid gone); dispatcher will retry", None, None),
"timed_out": lambda ev, n: (
f"⏱ {n.head} timed out (max_runtime={int(_payload(ev, 'limit_seconds') or 0)}s); will retry", None, None,
),
"status": lambda ev, n: (f"🔄 {n.head} → {_payload(ev, 'status') or ''}", None, None),
"review_requested": _fmt_review_requested,
"changes_requested": _fmt_changes_requested,
"block_loop_detected": _fmt_block_loop_detected,
# Re-blocked for the same cause past the limit and routed to `triage` for a
# human. It emits no blocked/status event, so ping loudly here.
"block_loop_detected": lambda ev, n: (
f"🛑 {n.head} routed to TRIAGE — needs a human decision"
f"{_clip(ev, 'recurrences', ' (blocked {}x for the same cause)', 200)}{_clip(ev, 'reason', ': {}', 160)}",
None, None,
),
}
# ---------------------------------------------------------------------------
# Delivery of one claimed batch (one subscription, N events)
# ---------------------------------------------------------------------------
# --- Delivery of one claimed batch (one subscription, N events) ---
class _KanbanNotification:
@@ -443,8 +326,8 @@ class _KanbanNotification:
self.title = (task.title if task else sub["task_id"])[:120]
self.board_tag = f"[{self.board_slug}] " if self.board_slug else ""
# Attribute the ping to the worker that did the work.
who = task.assignee if task and task.assignee else None
self.tag = f"@{who} " if who else ""
tag = f"@{task.assignee} " if task and task.assignee else ""
self.head = f"{self.board_tag}{tag}Kanban {self.task_id}"
# The wake self-post path needs the key even when every event was skipped.
self.sub_key = (sub["task_id"], sub["platform"], sub["chat_id"], sub.get("thread_id") or "")
mode = sub.get("delivery_mode") or "notify"
@@ -452,14 +335,11 @@ class _KanbanNotification:
self.send_passive = mode != "wake"
# Worker handoff carried into the synthetic wake turn so the woken
# creator doesn't re-decompose work already on the board.
self.wake_handoff = ""
self.wake_review_detail = ""
self.wake_handoff = self.wake_review_detail = self.session_key = self.synth = ""
self.plat: Any = None
self.adapter: Any = None
self.is_push_adapter = True
self.wake_kinds: set = set()
self.session_key = ""
self.synth = ""
# -- cursor / subscription ops (blocking, run in a fresh-context thread) --
@@ -490,11 +370,8 @@ class _KanbanNotification:
await self.rewind()
async def _wake_failed(self, fmt: str, exc: Exception) -> None:
await self.delivery_failed(
fmt, (self.task_id,),
"kanban notifier: dropping subscription %s on %s after %d consecutive wake failures",
exc, True,
)
drop_fmt = "kanban notifier: dropping subscription %s on %s after %d consecutive wake failures"
await self.delivery_failed(fmt, (self.task_id,), drop_fmt, exc, True)
# -- formatting --
@@ -513,11 +390,7 @@ class _KanbanNotification:
def build_wake_text(self) -> None:
"""Set ``wake_kinds`` / ``session_key`` / ``synth`` for the wake paths."""
task, sub = self.task, self.sub
self.wake_kinds = (
{ev.kind for ev in self.d["events"] if ev.kind in _WAKE_KINDS}
if self.wake_agent
else set()
)
self.wake_kinds = {ev.kind for ev in self.d["events"] if ev.kind in _WAKE_KINDS} if self.wake_agent else set()
if not self.wake_kinds:
return
if self.is_push_adapter:
@@ -532,11 +405,8 @@ class _KanbanNotification:
_status = t("gateway.kanban.wake.status_joiner").join(_parts) or t("gateway.kanban.wake.status_default")
synth = t(
"gateway.kanban.wake.message",
task_id=sub["task_id"],
status=_status,
title=self.title,
assignee=task.assignee if task else "",
board=self.board_slug,
task_id=sub["task_id"], status=_status, title=self.title,
assignee=task.assignee if task else "", board=self.board_slug,
)
# Label as an automatic notification and carry the handoff so the
# creator inspects the board instead of re-decomposing.
@@ -547,36 +417,32 @@ class _KanbanNotification:
self.synth = synth + "\n\n" + t("gateway.kanban.wake.guidance")
def _log_woke(self) -> None:
logger.info(
"kanban notifier: woke agent for %s on %s/%s profile=%s events=%s",
self.task_id, self.platform_str, self.sub["chat_id"], self.sub_profile or "default", self.wake_kinds,
)
logger.info("kanban notifier: woke agent for %s on %s/%s profile=%s events=%s",
self.task_id, self.platform_str, self.sub["chat_id"], self.sub_profile or "default", self.wake_kinds)
async def push_wake(self) -> None:
"""Wake the creator session behind a push adapter; raises on failure."""
from gateway.session import SessionSource
async def wake(self) -> None:
"""Wake the creator session (raises on failure): push adapters get a full SessionSource, non-push a raw self-post."""
from gateway.wake import deliver_wake
sub = self.sub
# Rebuild the creator's real session scope from the persisted
# chat_type: build_session_key() keys DMs differently from
# group/thread, so a hardcoded "group" mis-routed DM/thread creators
# into a fresh session. Legacy rows may carry chat_type in
# delivery_metadata; last resort is "group". A mismatch only degrades
# to a fresh session.
if not self.is_push_adapter:
await deliver_wake(self.adapter, text=self.synth, session_id=self.session_key)
self._log_woke()
return
from gateway.session import SessionSource
# Rebuild the creator's real session scope from the persisted chat_type:
# build_session_key() keys DMs differently from group/thread, so a
# hardcoded "group" mis-routed DM/thread creators into a fresh session.
# Legacy rows may carry chat_type in delivery_metadata; last resort is
# "group". A mismatch only degrades to a fresh session.
_chat_type = str(sub.get("chat_type") or "").strip()
if not _chat_type:
_delivery_meta = sub.get("delivery_metadata")
if isinstance(_delivery_meta, dict):
_chat_type = str(_delivery_meta.get("chat_type") or "").strip()
_source = SessionSource(
platform=self.plat,
chat_id=sub["chat_id"],
chat_type=_chat_type or "group",
thread_id=sub.get("thread_id") or None,
user_id=sub.get("user_id"),
user_id_alt=sub.get("user_id_alt"),
profile=self.sub_profile or None,
scope_id=_wake_scope_id(self.adapter, sub),
platform=self.plat, chat_id=sub["chat_id"], chat_type=_chat_type or "group",
thread_id=sub.get("thread_id") or None, user_id=sub.get("user_id"), user_id_alt=sub.get("user_id_alt"),
profile=self.sub_profile or None, scope_id=_wake_scope_id(self.adapter, sub),
)
await deliver_wake(self.adapter, text=self.synth, session_id=self.session_key, source=_source)
self._log_woke()
@@ -593,30 +459,19 @@ class _KanbanNotification:
# (else the event is lost); None / non-SendResult keeps the
# "no exception == delivered" contract.
if getattr(_send_res, "success", True) is False:
raise RuntimeError(
"adapter send() reported failure: "
f"{getattr(_send_res, 'error', None) or 'unknown error'}"
)
logger.debug(
"kanban notifier: delivered %s event for %s to %s/%s on board %s",
ev.kind, self.task_id, self.platform_str, sub["chat_id"], self.board_slug,
)
raise RuntimeError(f"adapter send() reported failure: {getattr(_send_res, 'error', None) or 'unknown error'}")
logger.debug("kanban notifier: delivered %s event for %s to %s/%s on board %s",
ev.kind, self.task_id, self.platform_str, sub["chat_id"], self.board_slug)
# Upload artifact paths from the completion payload / legacy result as
# native files. Only on ``completed`` so retries never spam attachments.
if ev.kind == "completed":
try:
await self.runner._deliver_kanban_artifacts(
adapter=adapter,
chat_id=sub["chat_id"],
metadata=metadata,
event_payload=getattr(ev, "payload", None),
task=self.task,
adapter=adapter, chat_id=sub["chat_id"], metadata=metadata,
event_payload=getattr(ev, "payload", None), task=self.task,
)
except Exception as art_exc:
logger.debug(
"kanban notifier: artifact delivery for %s failed: %s",
self.task_id, art_exc,
)
logger.debug("kanban notifier: artifact delivery for %s failed: %s", self.task_id, art_exc)
async def _send_pings(self) -> bool:
"""Send every text ping; False when a send failed (claim already rewound/dropped)."""
@@ -630,10 +485,8 @@ class _KanbanNotification:
# IS the delivery and resolves the failure counter.
if not self.is_push_adapter and self.wake_agent:
logger.debug(
"kanban notifier: adapter %s has no push "
"channel; skipping text ping for %s, relying "
"on wake self-post instead",
self.platform_str, self.task_id,
"kanban notifier: adapter %s has no push channel; skipping text ping for %s, relying "
"on wake self-post instead", self.platform_str, self.task_id,
)
continue
if not self.send_passive:
@@ -644,10 +497,8 @@ class _KanbanNotification:
self.clear_failures()
except Exception as exc:
await self.delivery_failed(
"kanban notifier: send failed for %s on %s (attempt %d/%d): %s",
(self.task_id, self.platform_str),
"kanban notifier: dropping subscription %s on %s after %d consecutive send failures",
exc, False,
"kanban notifier: send failed for %s on %s (attempt %d/%d): %s", (self.task_id, self.platform_str),
"kanban notifier: dropping subscription %s on %s after %d consecutive send failures", exc, False,
)
return False
return True
@@ -664,10 +515,8 @@ class _KanbanNotification:
# default) has no adapter.
adapter = self.runner._authorization_adapter(self.plat, self.sub_profile or None)
if adapter is None:
logger.debug(
"kanban notifier: adapter %s disconnected before delivery for %s; rewinding claim",
self.platform_str, self.task_id,
)
logger.debug("kanban notifier: adapter %s disconnected before delivery for %s; rewinding claim",
self.platform_str, self.task_id)
await self.rewind()
return
self.adapter = adapter
@@ -677,29 +526,21 @@ class _KanbanNotification:
if not await self._send_pings():
return
# All text pings delivered (or skipped for non-push / wake-only).
task_terminal = self.task and self.task.status == "archived"
self.build_wake_text()
wake_kinds, is_push = self.wake_kinds, self.is_push_adapter
if not is_push and wake_kinds and self.session_key:
# Self-post IS the delivery: must succeed BEFORE the cursor advances.
from gateway.wake import deliver_wake
# Non-push self-post, or wake-only push sub: the wake IS the delivery
# and must succeed BEFORE the cursor advances.
if wake_kinds and (not self.send_passive if is_push else bool(self.session_key)):
try:
await deliver_wake(adapter, text=self.synth, session_id=self.session_key)
self._log_woke()
await self.wake()
self.clear_failures()
except Exception as _wk_err:
await self._wake_failed("kanban notifier: wake self-post failed for %s (attempt %d/%d): %s", _wk_err)
return
if is_push and not self.send_passive and wake_kinds:
# Wake-only push sub: the wake is the sole delivery and must
# succeed BEFORE the cursor advances.
try:
await self.push_wake()
self.clear_failures()
except Exception as _wk_err:
await self._wake_failed("kanban notifier: wake-only delivery failed for %s (attempt %d/%d): %s", _wk_err)
await self._wake_failed(
"kanban notifier: wake-only delivery failed for %s (attempt %d/%d): %s" if is_push
else "kanban notifier: wake self-post failed for %s (attempt %d/%d): %s",
_wk_err,
)
return
# Delivery complete: advance the cursor (the dedup mechanism).
@@ -711,12 +552,9 @@ class _KanbanNotification:
# advanced; the wake stays best-effort, but log at WARNING so a
# persistently failing wake is visible.
try:
await self.push_wake()
await self.wake()
except Exception as _wk_err:
logger.warning(
"kanban notifier: wakeup injection failed for %s: %s",
self.task_id, _wk_err, exc_info=True,
)
logger.warning("kanban notifier: wakeup injection failed for %s: %s", self.task_id, _wk_err, exc_info=True)
# Unsubscribe only on archive; ``done`` is reversible.
if task_terminal:
if self.task and self.task.status == "archived":
await self.unsub()
gateway/lifecycle_ledger.py
+114 -201
View File
@@ -1,24 +1,12 @@
"""Gateway lifecycle ledger — durable termination-reason evidence.
Graceful shutdowns already leave forensics (:mod:`gateway.shutdown_forensics`,
``gateway-exit-diag.log``); an **unclean death** (SIGKILL, kernel OOM kill, VM
death) runs no handler, so the next boot would otherwise have no record of it.
A tiny state machine persisted to ``<HERMES_HOME>/state/gateway.lifecycle.json``
closes that gap:
* :func:`record_startup` reads the previous life's sentinel. ``phase ==
"running"`` means no exit path ran → unclean death. The finding (with the
last heartbeat's memory sample) is appended to ``gateway-exit-diag.log`` as a
``gateway.previous_unclean_exit`` record and logged at WARNING, then the
sentinel is rewritten as ``phase=running`` for the new life.
* :func:`mark_exited` rewrites the sentinel as ``phase=exited`` on every clean
exit path (``_exit_after_graceful_shutdown`` and the watchdog ``os._exit`` sites).
* :func:`sample_memory` is the cheap /proc snapshot the 30s loop heartbeat
embeds, so OOM crash cycles are classifiable from the volume alone.
Everything here is best-effort: a forensics failure must never affect the
gateway lifecycle it is observing.
Graceful shutdowns leave forensics (``gateway-exit-diag.log``); an unclean
death (SIGKILL, kernel OOM, VM death) runs no handler. A sentinel at
``<HERMES_HOME>/state/gateway.lifecycle.json`` closes the gap:
:func:`record_startup` finds ``phase == "running"`` from the previous life →
unclean death, appended to the exit-diag log as ``gateway.previous_unclean_exit``
and logged at WARNING; :func:`mark_exited` rewrites ``phase=exited`` on every
clean exit path. Best-effort: forensics must never affect the lifecycle.
"""
from __future__ import annotations
@@ -28,74 +16,61 @@ import logging
import os
import sqlite3
import time
from contextlib import closing
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Dict, Optional
logger = logging.getLogger(__name__)
_LIFECYCLE_RELATIVE = ("state", "gateway.lifecycle.json")
_EXIT_DIAG_RELATIVE = ("logs", "gateway-exit-diag.log")
_STATE_DB_RELATIVE = ("state.db",)
# Heuristic OOM-suspicion thresholds on the last heartbeat's memory sample.
# Deliberately conservative: only a hint; classification stays with the reader.
_LOW_MEM_AVAILABLE_KIB = 64 * 1024 # < 64 MiB available
_LOW_MEM_AVAILABLE_FRACTION = 0.05 # < 5% of MemTotal available
def _process_hermes_home() -> Path:
"""HERMES_HOME for process-level identity files (ignore task overrides)."""
val = os.environ.get("HERMES_HOME", "").strip()
if val:
return Path(val)
from hermes_constants import get_hermes_home
return get_hermes_home()
val = os.environ.get("HERMES_HOME", "").strip()
return Path(val) if val else get_hermes_home()
def _home_path(home: Optional[Path], relative: tuple) -> Path:
base = home if home is not None else _process_hermes_home()
return base.joinpath(*relative)
def _home_path(home: Optional[Path], *relative: str) -> Path:
return (_process_hermes_home() if home is None else home).joinpath(*relative)
def get_lifecycle_sentinel_path(home: Optional[Path] = None) -> Path:
"""Return ``<HERMES_HOME>/state/gateway.lifecycle.json``."""
return _home_path(home, _LIFECYCLE_RELATIVE)
return _home_path(home, "state", "gateway.lifecycle.json")
def sample_memory() -> Dict[str, Any]:
"""Cheap memory snapshot: own RSS + system availability + swap.
def _now_iso() -> str:
return datetime.now(timezone.utc).isoformat()
Pure ``/proc`` reads, Linux-only (returns ``{}`` elsewhere), never
raises. Values in KiB to match the kernel's units.
"""
sample: Dict[str, Any] = {}
def _proc_fields(path: str, wanted: Dict[str, str]) -> Dict[str, int]:
"""``{dst: int}`` for each ``src: dst`` key found in a ``Key: value`` /proc file."""
found: Dict[str, int] = {}
try:
with open("/proc/self/status", encoding="utf-8") as fh:
for line in fh:
if line.startswith("VmRSS:"):
sample["rss_kib"] = int(line.split()[1])
break
except (OSError, ValueError, IndexError):
pass
try:
meminfo: Dict[str, int] = {}
wanted = {"MemTotal", "MemAvailable", "SwapTotal", "SwapFree"}
with open("/proc/meminfo", encoding="utf-8") as fh:
with open(path, encoding="utf-8") as fh:
for line in fh:
key = line.split(":", 1)[0]
if key in wanted:
meminfo[key] = int(line.split()[1])
if len(meminfo) == len(wanted):
found[wanted[key]] = int(line.split()[1])
if len(found) == len(wanted):
break
for src, dst in (("MemTotal", "mem_total_kib"), ("MemAvailable", "mem_available_kib")):
if src in meminfo:
sample[dst] = meminfo[src]
if "SwapTotal" in meminfo and "SwapFree" in meminfo:
sample["swap_used_kib"] = meminfo["SwapTotal"] - meminfo["SwapFree"]
except (OSError, ValueError, IndexError):
pass
return {}
return found
def sample_memory() -> Dict[str, Any]:
"""Cheap /proc snapshot (KiB): own RSS + MemTotal/MemAvailable + swap used. Linux-only
(``{}`` elsewhere), never raises; the 30s heartbeat embeds it so OOM cycles are classifiable."""
sample = _proc_fields("/proc/self/status", {"VmRSS": "rss_kib"})
mem = _proc_fields("/proc/meminfo", {"MemTotal": "mem_total_kib", "MemAvailable": "mem_available_kib",
"SwapTotal": "SwapTotal", "SwapFree": "SwapFree"})
swap_total, swap_free = mem.pop("SwapTotal", None), mem.pop("SwapFree", None)
sample.update(mem)
if swap_total is not None and swap_free is not None:
sample["swap_used_kib"] = swap_total - swap_free
return sample
@@ -108,10 +83,10 @@ def _read_json(path: Path) -> Optional[Dict[str, Any]]:
def _write_sentinel(payload: Dict[str, Any], home: Optional[Path]) -> None:
path = get_lifecycle_sentinel_path(home)
try:
from utils import atomic_json_write
path = get_lifecycle_sentinel_path(home)
path.parent.mkdir(parents=True, exist_ok=True)
atomic_json_write(path, payload, indent=None)
except Exception:
@@ -119,10 +94,9 @@ def _write_sentinel(payload: Dict[str, Any], home: Optional[Path]) -> None:
def _append_exit_diag(record: Dict[str, Any], home: Optional[Path]) -> None:
"""Append a JSON line to gateway-exit-diag.log (same format as the CLI's
``_exit_diag`` records so existing tooling greps both)."""
path = _home_path(home, _EXIT_DIAG_RELATIVE)
"""Append a JSON line to gateway-exit-diag.log (same format as the CLI's ``_exit_diag``)."""
try:
path = _home_path(home, "logs", "gateway-exit-diag.log")
path.parent.mkdir(parents=True, exist_ok=True)
with path.open("a", encoding="utf-8") as fh:
fh.write(json.dumps(record, default=str) + "\n")
@@ -131,158 +105,118 @@ def _append_exit_diag(record: Dict[str, Any], home: Optional[Path]) -> None:
def _pid_alive_with_start_time(pid: Any, start_time: Any) -> bool:
"""True when ``pid`` is a live process matching ``start_time`` (±2s).
Guards the takeover race: during ``--replace`` the old gateway can still
be mid-teardown when the new one boots — a live matching owner is a
planned handover, not an unclean death.
"""
"""True when ``pid`` is a live process matching ``start_time`` (±2s) — guards the
``--replace`` race: a live matching owner mid-teardown is a handover, not a death."""
try:
pid_int = int(pid)
except (TypeError, ValueError):
return False
if pid_int <= 0:
return False
try:
# NOT os.kill(pid, 0): on Windows that sends CTRL_C_EVENT to the
# target's console group. _pid_exists is the canonical no-kill probe.
from gateway.status import _pid_exists
# NOT os.kill(pid, 0): on Windows that sends CTRL_C_EVENT to the target's console group.
from gateway.status import _pid_exists, get_process_start_time
if not _pid_exists(pid_int):
if pid_int <= 0 or not _pid_exists(pid_int):
return False
except Exception:
return False
if start_time is None:
return True # alive; can't disambiguate PID reuse — err on "alive"
try:
from gateway.status import get_process_start_time
actual = get_process_start_time(pid_int)
return actual is None or abs(float(actual) - float(start_time)) <= 2.0
actual = None if start_time is None else get_process_start_time(pid_int)
return actual is None or abs(float(actual) - float(start_time)) <= 2.0 # None: can't disambiguate PID reuse
except Exception:
return True
def _suspected_oom(mem: Dict[str, Any]) -> bool:
"""Heuristic only (classification stays with the reader); thresholds are
memory_status' "critical" tier so a live warning and a post-mortem verdict agree."""
from gateway.memory_status import _CRITICAL_AVAILABLE_FRACTION, _CRITICAL_AVAILABLE_KIB
total, avail = mem.get("mem_total_kib"), mem.get("mem_available_kib")
if not isinstance(avail, int):
return False
return avail < _CRITICAL_AVAILABLE_KIB or (
isinstance(total, int) and total > 0 and avail / total < _CRITICAL_AVAILABLE_FRACTION
)
def detect_unclean_exit(home: Optional[Path] = None) -> Optional[Dict[str, Any]]:
"""Inspect the previous life's sentinel; return an evidence dict when it
died uncleanly, else ``None``. Read-only — does not rewrite the sentinel.
"""
"""Evidence dict when the previous life died uncleanly, else ``None``. Read-only."""
sentinel = _read_json(get_lifecycle_sentinel_path(home))
if not sentinel or sentinel.get("phase") != "running":
return None
if _pid_alive_with_start_time(sentinel.get("pid"), sentinel.get("start_time")):
return None # live owner — planned takeover in flight, not a death
evidence: Dict[str, Any] = {
"prior_pid": sentinel.get("pid"),
"prior_started_at": sentinel.get("started_at"),
"prior_pid": sentinel.get("pid"), "prior_started_at": sentinel.get("started_at"),
"prior_start_time": sentinel.get("start_time"),
}
# Enrich with the last heartbeat: last proven liveness and memory at that moment.
try:
from gateway.shutdown_watchdog import get_loop_heartbeat_path
hb = _read_json(get_loop_heartbeat_path(home))
hb = _read_json(get_loop_heartbeat_path(home)) or {}
except Exception:
hb = None
hb = {}
if hb:
evidence["last_heartbeat_at"] = hb.get("updated_at")
mem = hb.get("mem")
if isinstance(mem, dict):
evidence["last_heartbeat_mem"] = mem
total = mem.get("mem_total_kib")
avail = mem.get("mem_available_kib")
if isinstance(avail, int) and (
avail < _LOW_MEM_AVAILABLE_KIB
or (
isinstance(total, int)
and total > 0
and avail / total < _LOW_MEM_AVAILABLE_FRACTION
)
):
evidence["suspected_oom"] = True
mem = hb.get("mem")
if isinstance(mem, dict):
evidence["last_heartbeat_mem"] = mem
if _suspected_oom(mem):
evidence["suspected_oom"] = True
return evidence
def check_state_db_integrity(home: Optional[Path] = None) -> str:
"""Return ``"ok"``, ``"absent"``, or the first ``quick_check`` complaint.
"""``"ok"``, ``"absent"``, or the first ``quick_check`` complaint. Never raises.
Called only after an unclean death — a SIGKILL mid-WAL-checkpoint can leave
half-written b-tree pages (see ``_enforce_macos_synchronous_full`` in
:mod:`hermes_state`). ``quick_check(1)`` stops at the first problem (~2s on
a healthy 500MB store): cheap once per unclean boot, too costly every boot.
Opened normally, not read-only: a WAL store needs its -shm sidecar for a
read-only open, and the PRAGMA itself writes nothing. Never raises.
Only after an unclean death — SIGKILL mid-WAL-checkpoint can leave half-written
b-tree pages. ``quick_check(1)`` stops at the first problem (~2s on a healthy
500MB store): cheap once per unclean boot, too costly every boot. Opened
normally: a WAL store needs its -shm sidecar for read-only, and the PRAGMA writes nothing.
"""
path = _home_path(home, _STATE_DB_RELATIVE)
path = _home_path(home, "state.db")
if not path.exists():
return "absent"
try:
conn = sqlite3.connect(str(path))
try:
with closing(sqlite3.connect(str(path))) as conn:
row = conn.execute("PRAGMA quick_check(1)").fetchone()
finally:
conn.close()
except Exception as exc: # sqlite3.Error, OSError, anything
except Exception as exc:
return f"check-failed: {exc}"
if not row or row[0] is None:
return "check-failed: no result"
return str(row[0])
return "check-failed: no result" if not row or row[0] is None else str(row[0])
def _report_unclean_exit(evidence: Dict[str, Any], home: Optional[Path]) -> None:
"""Integrity-check the store, persist the exit-diag record, log at WARNING."""
# The death may have torn the store; this is the only moment we know to look.
verdict = evidence["state_db_integrity"] = check_state_db_integrity(home=home)
if verdict not in ("ok", "absent"):
logger.error(
"state.db FAILED integrity check after an unclean gateway exit: %s — sessions may read as "
"missing until it is repaired. Run `hermes doctor`.",
verdict,
)
_append_exit_diag({"ts": _now_iso(), "tag": "gateway.previous_unclean_exit", "pid": os.getpid(), **evidence}, home)
logger.warning(
"Previous gateway life (pid=%s, started_at=%s) exited UNCLEANLY (no exit path ran — SIGKILL / OOM / "
"VM death). last_heartbeat_at=%s last_mem=%s suspected_oom=%s",
evidence.get("prior_pid"), evidence.get("prior_started_at"), evidence.get("last_heartbeat_at"),
evidence.get("last_heartbeat_mem"), evidence.get("suspected_oom", False),
)
def record_startup(home: Optional[Path] = None) -> Optional[Dict[str, Any]]:
"""Boot-time entry point: report any unclean previous exit, then claim
the sentinel for the current life.
Returns the unclean-exit evidence dict (also persisted to
``gateway-exit-diag.log`` and logged at WARNING) or ``None``. Never
raises.
"""
"""Boot entry point: report any unclean previous exit (evidence dict, also persisted
to ``gateway-exit-diag.log`` and logged at WARNING) then claim the sentinel. Never raises."""
evidence: Optional[Dict[str, Any]] = None
try:
evidence = detect_unclean_exit(home)
if evidence is not None:
# The death may have torn the store; this is the only moment we know to look.
verdict = check_state_db_integrity(home=home)
evidence["state_db_integrity"] = verdict
if verdict not in ("ok", "absent"):
logger.error(
"state.db FAILED integrity check after an unclean gateway "
"exit: %s — sessions may read as missing until it is "
"repaired. Run `hermes doctor`.",
verdict,
)
_append_exit_diag({
"ts": datetime.now(timezone.utc).isoformat(),
"tag": "gateway.previous_unclean_exit",
"pid": os.getpid(),
**evidence,
}, home)
logger.warning(
"Previous gateway life (pid=%s, started_at=%s) exited UNCLEANLY "
"(no exit path ran — SIGKILL / OOM / VM death). "
"last_heartbeat_at=%s last_mem=%s suspected_oom=%s",
evidence.get("prior_pid"),
evidence.get("prior_started_at"),
evidence.get("last_heartbeat_at"),
evidence.get("last_heartbeat_mem"),
evidence.get("suspected_oom", False),
)
_report_unclean_exit(evidence, home)
except Exception:
logger.debug("Unclean-exit detection failed", exc_info=True)
try:
claim: Dict[str, Any] = {
"phase": "running",
"pid": os.getpid(),
"start_time": time.time(),
"started_at": datetime.now(timezone.utc).isoformat(),
}
# Carry the verdict on the PREVIOUS life forward on the new sentinel: it
# is the only machine-readable copy (the exit-diag log is append-only
# prose) and /api/status reads it to report an OOM restart. Scoped to
# this life only — the next clean exit or boot rewrites the sentinel.
claim: Dict[str, Any] = {"phase": "running", "pid": os.getpid(), "start_time": time.time(), "started_at": _now_iso()}
# Carry the verdict on the PREVIOUS life on the new sentinel: it is the only
# machine-readable copy (/api/status reads it to report an OOM restart).
# Scoped to this life — the next clean exit or boot rewrites the sentinel.
if evidence is not None:
claim["prior_unclean_exit"] = True
if evidence.get("suspected_oom"):
@@ -293,49 +227,28 @@ def record_startup(home: Optional[Path] = None) -> Optional[Dict[str, Any]]:
return evidence
def mark_exited(
exit_code: Optional[int] = None,
reason: str = "graceful_shutdown",
home: Optional[Path] = None,
) -> None:
def mark_exited(exit_code: Optional[int] = None, reason: str = "graceful_shutdown", home: Optional[Path] = None) -> None:
"""Mark the current life as cleanly exited. Idempotent, never raises.
Only rewrites the sentinel when it is provably owned by this process:
during a ``--replace`` takeover the replacement claims the sentinel before
the old process finishes teardown, and the old life must not clobber the
new owner's ``running`` phase. A sentinel with ``pid=None`` (or malformed)
has unknown ownership and is likewise left alone.
Only rewrites a sentinel provably owned by this process: during ``--replace``
the replacement claims it before the old process finishes teardown and must not
be clobbered. ``pid=None`` / malformed sentinels have unknown ownership → left alone.
"""
try:
sentinel = _read_json(get_lifecycle_sentinel_path(home))
if sentinel is not None and sentinel.get("pid") != os.getpid():
return
_write_sentinel({
"phase": "exited",
"pid": os.getpid(),
"exit_code": exit_code,
"exit_reason": reason,
"exited_at": datetime.now(timezone.utc).isoformat(),
}, home)
_write_sentinel({"phase": "exited", "pid": os.getpid(), "exit_code": exit_code, "exit_reason": reason,
"exited_at": _now_iso()}, home)
except Exception:
logger.debug("Failed to mark lifecycle sentinel exited", exc_info=True)
def read_prior_exit_label(profile_home: Path) -> str:
"""Container-boot helper: ``clean`` / ``unclean`` / ``unknown`` summary of
how the profile's last gateway life ended. Read-only, exception-free;
used by ``hermes_cli.container_boot`` to annotate ``container-boot.log``.
"""
"""``clean``/``unclean``/``unknown`` for the profile's last gateway life; annotates
``container-boot.log``. ``running`` is unclean: the old PID namespace is gone at container boot."""
try:
sentinel = _read_json(get_lifecycle_sentinel_path(profile_home))
if not sentinel:
return "unknown"
phase = sentinel.get("phase")
if phase == "exited":
return "clean"
if phase == "running":
# Old PID namespace is gone at container boot — never exited cleanly.
return "unclean"
phase = (_read_json(get_lifecycle_sentinel_path(profile_home)) or {}).get("phase")
return {"exited": "clean", "running": "unclean"}.get(phase, "unknown")
except Exception:
pass
return "unknown"
return "unknown"
gateway/media_policy.py
+12 -28
View File
@@ -1,21 +1,12 @@
"""Shared config→env bridge for media-delivery policy.
``validate_media_delivery_path`` (gateway/platforms/base.py) reads its policy
from environment variables:
- ``HERMES_MEDIA_DELIVERY_STRICT`` <- gateway.strict
- ``HERMES_MEDIA_ALLOW_DIRS`` <- gateway.media_delivery_allow_dirs
- ``HERMES_MEDIA_TRUST_RECENT_FILES`` <- gateway.trust_recent_files
The translation used to run only in gateway startup, so standalone delivery
paths (``hermes cron run``, ``hermes send``, a standalone cron tick) filtered
MEDIA paths under a different policy and silently dropped attachments in
strict/allowlisted deployments (text is unaffected -- only media goes through
path validation). ``apply_media_policy_env()`` is the shared,
idempotent helper every delivery entrypoint calls before filtering media paths.
Precedence: an explicitly-set environment variable WINS over config.yaml, so a
shell-exported override (and gateway startup's own earlier run) survives.
``validate_media_delivery_path`` reads ``HERMES_MEDIA_DELIVERY_STRICT`` (gateway.strict),
``HERMES_MEDIA_ALLOW_DIRS`` (gateway.media_delivery_allow_dirs) and
``HERMES_MEDIA_TRUST_RECENT_FILES`` (gateway.trust_recent_files). Every delivery
entrypoint (gateway startup, ``hermes cron run``, ``hermes send``) calls
:func:`apply_media_policy_env` first so standalone paths filter under the gateway's
policy instead of silently dropping attachments in strict/allowlisted deployments.
An explicitly-set env var WINS over config.yaml, so shell overrides survive.
"""
from __future__ import annotations
@@ -26,9 +17,8 @@ from typing import Any, Dict, Optional
logger = logging.getLogger(__name__)
_STRICT_ENV = "HERMES_MEDIA_DELIVERY_STRICT"
_FLAG_ENVS = (("strict", "HERMES_MEDIA_DELIVERY_STRICT"), ("trust_recent_files", "HERMES_MEDIA_TRUST_RECENT_FILES"))
_ALLOW_DIRS_ENV = "HERMES_MEDIA_ALLOW_DIRS"
_TRUST_RECENT_ENV = "HERMES_MEDIA_TRUST_RECENT_FILES"
def _load_gateway_cfg(config: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
@@ -50,25 +40,19 @@ def _set_env_default(env: str, value: str) -> None:
def _allow_dirs_str(allow_dirs: Any) -> str:
if isinstance(allow_dirs, str):
return allow_dirs
if isinstance(allow_dirs, (list, tuple)):
return os.pathsep.join(str(p) for p in allow_dirs if p)
return ""
return allow_dirs if isinstance(allow_dirs, str) else ""
def apply_media_policy_env(config: Optional[Dict[str, Any]] = None) -> None:
"""Bridge gateway media-policy settings from config.yaml into the env.
Idempotent and env-wins: a variable already present is never overwritten.
Never raises — a policy-bridge failure must not break delivery; the
validator falls back to its defaults exactly as before.
"""
"""Bridge gateway media-policy settings from config.yaml into the env. Idempotent,
env-wins, never raises — a bridge failure must not break delivery (validator defaults apply)."""
try:
gateway_cfg = _load_gateway_cfg(config)
if not gateway_cfg:
return
for key, env in (("strict", _STRICT_ENV), ("trust_recent_files", _TRUST_RECENT_ENV)):
for key, env in _FLAG_ENVS:
flag = gateway_cfg.get(key)
if flag is not None:
_set_env_default(env, "1" if flag else "0")
gateway/media_repair.py
+59 -105
View File
@@ -1,19 +1,11 @@
"""Repair model-mangled ``computer_use`` screenshot paths in final responses.
``computer_use`` persists a screenshot into the Hermes image cache and tells the
model its absolute path. Some models rewrite a Windows path into a POSIX-looking
one (``C:\\Users\\Alice\\...`` -> ``/Users/Alice/...``) inside an explicit
``MEDIA:`` directive, so delivery-path validation rejects it and drops the
attachment.
The repair is deliberately narrow: it only rewrites paths inside a response that
*already* carries an explicit ``MEDIA:`` directive, and only when the directive's
generated ``computer_use_<uuid>`` basename exactly matches a canonical screenshot
path returned by ``computer_use`` in the current turn. It never auto-attaches
captures; normal media path validation still runs afterwards.
Own module (like ``gateway/media_policy.py``) so the gateway turn path, gateway
background tasks and cron delivery share one implementation.
Some models rewrite the Windows path ``computer_use`` reported into a POSIX-looking
one (``C:\\Users\\Alice\\...`` -> ``/Users/Alice/...``) inside an explicit ``MEDIA:``
directive, so delivery-path validation rejects it and drops the attachment.
Deliberately narrow: only rewrites paths in a response that *already* carries a
``MEDIA:`` directive whose ``computer_use_<uuid>`` basename exactly matches a
canonical path returned this turn. Never auto-attaches; validation still runs.
"""
from __future__ import annotations
@@ -29,15 +21,12 @@ logger = logging.getLogger(__name__)
# root, or UNC share. Shared string so the summary regex stays in sync.
_ABS_PATH_PREFIX_PATTERN = r"(?:[A-Za-z]:[/\\]|/|\\\\)"
_ABS_PATH_PREFIX_RE = re.compile(r"^" + _ABS_PATH_PREFIX_PATTERN)
_CAPTURE_BASENAME_PATTERN = r"computer_use_[0-9a-f]{32}\.(?:png|jpe?g)"
_COMPUTER_USE_CAPTURE_BASENAME_RE = re.compile(
r"^computer_use_[0-9a-f]{32}\.(?:png|jpe?g)$",
re.IGNORECASE,
)
_COMPUTER_USE_CAPTURE_BASENAME_RE = re.compile(r"^" + _CAPTURE_BASENAME_PATTERN + r"$", re.IGNORECASE)
_COMPUTER_USE_CAPTURE_SUMMARY_RE = re.compile(
r"\(shareable screenshot saved to "
r"(?P<path>" + _ABS_PATH_PREFIX_PATTERN + r"[^\r\n]*?"
r"computer_use_[0-9a-f]{32}\.(?:png|jpe?g))\)",
r"(?P<path>" + _ABS_PATH_PREFIX_PATTERN + r"[^\r\n]*?" + _CAPTURE_BASENAME_PATTERN + r")\)",
re.IGNORECASE,
)
@@ -50,8 +39,7 @@ def tool_name_by_call_id(messages: List[Dict[str, Any]]) -> Dict[str, str]:
continue
for call in msg.get("tool_calls") or []:
call_id = call.get("id") or call.get("call_id")
fn = call.get("function") or {}
name = str(fn.get("name") or call.get("name") or "")
name = str((call.get("function") or {}).get("name") or call.get("name") or "")
if call_id and name:
mapping[str(call_id)] = name
return mapping
@@ -59,130 +47,96 @@ def tool_name_by_call_id(messages: List[Dict[str, Any]]) -> Dict[str, str]:
def _computer_use_capture_basename(path: Any) -> str:
"""Canonical (lowercased) capture basename for either separator style, or ''."""
value = str(path or "").strip().strip("`\"'")
basename = re.split(r"[/\\]", value)[-1]
basename = re.split(r"[/\\]", str(path or "").strip().strip("`\"'"))[-1]
return basename.lower() if _COMPUTER_USE_CAPTURE_BASENAME_RE.fullmatch(basename) else ""
def _iter_computer_use_capture_paths(content: Any) -> Iterator[str]:
"""Yield persisted screenshot paths from computer_use result content.
The tool can return JSON, a multimodal content list, or a text fallback; the
latter two keep the canonical path in the human-readable summary even though
the multimodal envelope's ``meta`` is not stored in the tool message.
"""
"""Yield persisted screenshot paths from computer_use result content (JSON, a
multimodal list, or text; the latter two keep the canonical path in the summary
line since the envelope's ``meta`` is not stored in the tool message)."""
if isinstance(content, str):
stripped = content.strip()
if stripped.startswith(("{", "[")):
# Parse JSON first, never regex-scan it: JSON escaping doubles
# backslashes, so a raw-text hit would yield a path that exists
# nowhere. Fail closed on unparseable (truncated) JSON.
try:
payload = json.loads(stripped)
except Exception:
return
if isinstance(payload, (dict, list)):
yield from _iter_computer_use_capture_paths(payload)
if not stripped.startswith(("{", "[")):
for match in _COMPUTER_USE_CAPTURE_SUMMARY_RE.finditer(content):
yield match.group("path").strip()
return
for match in _COMPUTER_USE_CAPTURE_SUMMARY_RE.finditer(content):
yield match.group("path").strip()
return
if isinstance(content, list):
# Parse JSON, never regex-scan it: JSON escaping doubles backslashes, so a raw-text
# hit would yield a path that exists nowhere. Fail closed on truncated JSON.
try:
payload = json.loads(stripped)
except Exception:
return
if isinstance(payload, (dict, list)):
yield from _iter_computer_use_capture_paths(payload)
elif isinstance(content, list):
for part in content:
yield from _iter_computer_use_capture_paths(part)
return
if not isinstance(content, dict):
return
screenshot_path = content.get("screenshot_path")
if isinstance(screenshot_path, str):
yield screenshot_path
meta = content.get("meta")
if isinstance(meta, dict) and isinstance(meta.get("screenshot_path"), str):
yield meta["screenshot_path"]
# Producer shapes (tools/computer_use/tool.py::_capture_response):
# content/text = multimodal parts; text_summary/summary = the line carrying
# "(shareable screenshot saved to ...)".
for field in ("content", "text", "text_summary", "summary"):
nested = content.get(field)
if isinstance(nested, (str, dict, list)):
yield from _iter_computer_use_capture_paths(nested)
elif isinstance(content, dict):
meta = content.get("meta")
for holder in (content, meta if isinstance(meta, dict) else {}):
if isinstance(holder.get("screenshot_path"), str):
yield holder["screenshot_path"]
# Producer shapes (computer_use ``_capture_response``): content/text = multimodal
# parts; text_summary/summary = the "(shareable screenshot saved to ...)" line.
for field in ("content", "text", "text_summary", "summary"):
nested = content.get(field)
if isinstance(nested, (str, dict, list)):
yield from _iter_computer_use_capture_paths(nested)
def _current_turn_messages(messages: List[Dict[str, Any]], history_offset: int) -> List[Dict[str, Any]]:
if not history_offset:
return messages
if len(messages) >= history_offset:
if not history_offset or len(messages) >= history_offset:
return messages[history_offset:]
# Compression can invalidate the slice boundary: recover the turn from its
# last user message, fail closed if none remains. Deliberately narrower than
# gateway/run.py::_collect_auto_append_media_tags' scan-everything fallback —
# that decides whether to ATTACH; this only rewrites paths already emitted.
# Compression can invalidate the slice boundary: recover the turn from its last user
# message, fail closed if none remains. Narrower than run.py's scan-everything fallback
# for auto-attach — that decides whether to ATTACH; this only rewrites emitted paths.
for index in range(len(messages) - 1, -1, -1):
if messages[index].get("role") == "user":
return messages[index:]
return []
def repair_explicit_computer_use_media_paths(
response: str,
messages: List[Dict[str, Any]],
history_offset: int = 0,
) -> str:
"""Recover model-mangled paths for explicitly requested screenshots.
Repairs only an already-explicit ``MEDIA:`` directive whose generated
basename case-insensitively matches a canonical screenshot path from this
turn. Fail-open: the repair is cosmetic, so any unexpected error returns
the response unchanged rather than aborting delivery.
"""
def repair_explicit_computer_use_media_paths(response: str, messages: List[Dict[str, Any]], history_offset: int = 0) -> str:
"""Recover model-mangled paths in explicit ``MEDIA:`` directives whose basename
matches (case-insensitively) a canonical screenshot path from this turn.
Fail-open: the repair is cosmetic, so any error returns the response unchanged."""
try:
return _repair_explicit_computer_use_media_paths_inner(
response, messages, history_offset
)
return _repair_explicit_computer_use_media_paths_inner(response, messages, history_offset)
except Exception:
logger.debug("computer_use media path repair failed", exc_info=True)
return response
def _repair_explicit_computer_use_media_paths_inner(
response: str,
messages: List[Dict[str, Any]],
history_offset: int = 0,
) -> str:
if "MEDIA:" not in response:
return response
turn_messages = _current_turn_messages(messages, history_offset)
def _canonical_capture_paths(turn_messages: List[Dict[str, Any]]) -> Dict[str, str]:
"""``{lowercase basename: absolute canonical path}`` from this turn's computer_use results."""
call_id_names = tool_name_by_call_id(turn_messages)
canonical_by_basename: Dict[str, str] = {}
canonical: Dict[str, str] = {}
for msg in turn_messages:
if msg.get("role") not in {"tool", "function"}:
continue
call_id = str(msg.get("tool_call_id") or msg.get("call_id") or "")
tool_name = str(
msg.get("name") or msg.get("tool_name") or call_id_names.get(call_id) or ""
)
if tool_name != "computer_use":
if str(msg.get("name") or msg.get("tool_name") or call_id_names.get(call_id) or "") != "computer_use":
continue
for path in _iter_computer_use_capture_paths(msg.get("content")):
basename = _computer_use_capture_basename(path)
if basename and _ABS_PATH_PREFIX_RE.match(path):
canonical_by_basename[basename] = path
canonical[basename] = path
return canonical
def _repair_explicit_computer_use_media_paths_inner(response: str, messages: List[Dict[str, Any]], history_offset: int = 0) -> str:
if "MEDIA:" not in response:
return response
canonical_by_basename = _canonical_capture_paths(_current_turn_messages(messages, history_offset))
if not canonical_by_basename:
return response
# Lazy: keeps `import gateway.media_repair` cheap for standalone cron
# processes that may never hit a MEDIA: response. No import cycle either way.
# Lazy: keeps the import cheap for standalone cron processes that may never hit MEDIA:.
from gateway.platforms.base import BasePlatformAdapter
media_files, _ = BasePlatformAdapter.extract_media(response)
repaired = response
for emitted_path, _is_voice in media_files:
for emitted_path, _is_voice in BasePlatformAdapter.extract_media(response)[0]:
canonical = canonical_by_basename.get(_computer_use_capture_basename(emitted_path))
if canonical and emitted_path != canonical:
repaired = repaired.replace(emitted_path, canonical)
gateway/memory_monitor.py
+21 -51
View File
@@ -1,15 +1,15 @@
"""Periodic process memory usage logging for the gateway.
Ported from cline/cline#10343 (src/standalone/memory-monitor.ts). Emits one
grep-friendly ``[MEMORY] ...`` line every N seconds (default 300) from a daemon
thread so slow leaks in the long-lived gateway show up as an RSS time series in
``agent.log`` / ``gateway.log``. A baseline snapshot is logged on start and a
final one on stop. Uses stdlib ``resource`` first, ``psutil`` as fallback
(Windows); if neither works the monitor warns once and stays disabled.
Emits one grep-friendly ``[MEMORY] ...`` line every N seconds (default 300)
from a daemon thread so slow leaks show up as an RSS time series in the logs.
A baseline snapshot is logged on start and a final one on stop. Uses stdlib
``resource`` first, ``psutil`` as fallback (Windows); if neither works the
monitor warns once and stays disabled.
"""
from __future__ import annotations
import contextlib
import gc
import logging
import os
@@ -17,7 +17,6 @@ import sys
import threading
import time
from typing import Optional
import contextlib
logger = logging.getLogger(__name__)
@@ -26,7 +25,6 @@ _BYTES_TO_MB = 1024 * 1024
_monitor_thread: Optional[threading.Thread] = None
_stop_event: Optional[threading.Event] = None
_start_time: Optional[float] = None
_interval_seconds: float = 300.0
_lock = threading.Lock()
@@ -36,8 +34,7 @@ def _get_rss_mb() -> Optional[int]:
import resource
# ru_maxrss is KB on Linux but bytes on macOS.
maxrss = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
return int(maxrss / (_BYTES_TO_MB if sys.platform == "darwin" else 1024))
return int(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / (_BYTES_TO_MB if sys.platform == "darwin" else 1024))
except Exception:
pass
try:
@@ -49,25 +46,12 @@ def _get_rss_mb() -> Optional[int]:
def log_memory_usage(prefix: str = "") -> None:
"""Log current memory usage as ``[MEMORY] [<prefix> ]rss=... gc=... threads=... uptime=...``.
Safe to call on-demand from any thread at lifecycle moments.
"""
"""Log ``[MEMORY] [<prefix> ]rss=... gc=... threads=... uptime=...``; safe from any thread."""
rss = _get_rss_mb()
uptime = int(time.monotonic() - _start_time) if _start_time else 0
try:
gc_counts = gc.get_count() # (gen0, gen1, gen2)
except Exception:
gc_counts = (0, 0, 0)
try:
thread_count = threading.active_count()
except Exception:
thread_count = 0
tag = f"{prefix} " if prefix else ""
rss_text = "unavailable" if rss is None else f"{rss}MB"
logger.info(
"[MEMORY] %srss=%s gc=%s threads=%d uptime=%ds", tag, rss_text, gc_counts, thread_count, uptime
"[MEMORY] %srss=%s gc=%s threads=%d uptime=%ds", f"{prefix} " if prefix else "",
"unavailable" if rss is None else f"{rss}MB", gc.get_count(), threading.active_count(),
int(time.monotonic() - _start_time) if _start_time else 0,
)
@@ -76,41 +60,32 @@ def _monitor_loop(stop_event: threading.Event, interval: float) -> None:
while not stop_event.wait(interval):
try:
log_memory_usage()
except Exception as e:
# Never let the monitor crash the gateway.
except Exception as e: # never let the monitor crash the gateway
logger.debug("Memory monitor iteration failed: %s", e)
def start_memory_monitoring(interval_seconds: float = 300.0) -> bool:
"""Start periodic memory logging in a daemon thread (baseline logged immediately).
Returns True if a fresh monitor was started; False if one is already running
or RSS introspection is unavailable (warned once).
"""
global _monitor_thread, _stop_event, _start_time, _interval_seconds
"""Start periodic logging in a daemon thread (baseline logged immediately). False if
already running or RSS introspection is unavailable (warned once)."""
global _monitor_thread, _stop_event, _start_time
with _lock:
if _monitor_thread is not None and _monitor_thread.is_alive():
return False
if _get_rss_mb() is None:
logger.warning(
"[MEMORY] Memory monitoring unavailable: neither resource.getrusage "
"nor psutil could read process RSS — skipping periodic logging.",
"[MEMORY] Memory monitoring unavailable: neither resource.getrusage nor psutil could read process RSS "
"— skipping periodic logging.",
)
return False
_start_time = time.monotonic()
_interval_seconds = float(interval_seconds)
_stop_event = threading.Event()
log_memory_usage(prefix="baseline")
_monitor_thread = threading.Thread(
target=_monitor_loop,
args=(_stop_event, _interval_seconds),
name="gateway-memory-monitor",
daemon=True,
target=_monitor_loop, args=(_stop_event, float(interval_seconds)), name="gateway-memory-monitor", daemon=True
)
_monitor_thread.start()
logger.info("[MEMORY] Periodic memory monitoring started (interval: %ds)", int(_interval_seconds))
logger.info("[MEMORY] Periodic memory monitoring started (interval: %ds)", int(interval_seconds))
return True
@@ -124,14 +99,9 @@ def stop_memory_monitoring(timeout: float = 2.0) -> None:
with contextlib.suppress(Exception):
log_memory_usage(prefix="shutdown")
_stop_event.set()
thread = _monitor_thread
_monitor_thread = None
_stop_event = None
# Join outside the lock so a stuck log call can't deadlock shutdown.
with contextlib.suppress(Exception):
thread, _monitor_thread, _stop_event = _monitor_thread, None, None
with contextlib.suppress(Exception): # join outside the lock so a stuck log call can't deadlock the stop path
thread.join(timeout=timeout)
logger.info("[MEMORY] Periodic memory monitoring stopped")
gateway/memory_status.py
+39 -84
View File
@@ -1,104 +1,74 @@
"""Memory status rollup for ``/api/status``.
Read side for memory-pressure signals the gateway already persists but only
logs: the 30s ``state/gateway.heartbeat`` (RSS + MemAvailable/MemTotal + swap,
from :func:`gateway.shutdown_watchdog.write_loop_heartbeat`) and the lifecycle
sentinel's ``suspected_oom`` flag (:func:`gateway.lifecycle_ledger.record_startup`).
Distills them into a compact block the dashboard SPA and the NAS availability
sweep consume — no new sampling, no IPC, two small file reads.
Public-safety: ``/api/status`` is unauthenticated (``PUBLIC_API_PATHS``), so this
block carries only coarse numbers (MB granularity), enums, and booleans — the
same disclosure class as ``active_agents`` / ``nous_session_valid``.
Best-effort and read-only: a missing/corrupt file degrades to
Read side for signals the gateway already persists: the 30s ``state/gateway.heartbeat``
(RSS + MemAvailable/MemTotal + swap) and the lifecycle sentinel's ``suspected_oom``
flag — two small file reads, no IPC. ``/api/status`` is unauthenticated, so only
coarse numbers (MB), enums and booleans. A missing/corrupt file degrades to
``pressure="unknown"`` rather than raising into the status endpoint.
"""
from __future__ import annotations
import logging
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Dict, Optional
logger = logging.getLogger(__name__)
# Thresholds on system MemAvailable. ``critical`` mirrors the lifecycle ledger's
# OOM-suspicion heuristics (_LOW_MEM_AVAILABLE_KIB / _LOW_MEM_AVAILABLE_FRACTION):
# a level that would make a later unclean death "suspected OOM" should already
# warn while the process is alive.
# Thresholds on system MemAvailable. ``critical`` doubles as the lifecycle
# ledger's OOM-suspicion heuristic: a level that makes a later unclean death
# "suspected OOM" already warns while the process is alive.
_CRITICAL_AVAILABLE_KIB = 64 * 1024 # < 64 MiB available
_CRITICAL_AVAILABLE_FRACTION = 0.05 # < 5% of MemTotal
_ELEVATED_AVAILABLE_KIB = 128 * 1024 # < 128 MiB available
_ELEVATED_AVAILABLE_FRACTION = 0.15 # < 15% of MemTotal
_PRESSURE_TIERS = ( # order-sensitive: worst first
("critical", _CRITICAL_AVAILABLE_KIB, _CRITICAL_AVAILABLE_FRACTION),
("elevated", _ELEVATED_AVAILABLE_KIB, _ELEVATED_AVAILABLE_FRACTION),
)
# Writer cadence is 30s (DEFAULT_HEARTBEAT_INTERVAL_S); 150s tolerates a briefly
# stalled loop without letting a long-dead gateway's last sample pose as current.
# Writer cadence is 30s; 150s tolerates a briefly stalled loop without letting
# a long-dead gateway's last sample pose as current.
_HEARTBEAT_FRESH_TTL_S = 150.0
_KIB_PER_MB = 1024
def _nonneg_int(value: Any) -> Optional[int]:
"""Return *value* if it is a non-negative int (bools rejected), else None."""
if isinstance(value, bool) or not isinstance(value, int) or value < 0:
return None
return value
return value if isinstance(value, int) and not isinstance(value, bool) and value >= 0 else None
def _mb(kib: Any) -> Optional[int]:
kib = _nonneg_int(kib)
return None if kib is None else kib // _KIB_PER_MB
return None if _nonneg_int(kib) is None else kib // 1024
def _parse_iso(value: Any) -> Optional[datetime]:
if not isinstance(value, str) or not value:
return None
try:
parsed = datetime.fromisoformat(value)
parsed = datetime.fromisoformat(value) if isinstance(value, str) and value else None
except ValueError:
return None
return parsed if parsed.tzinfo is not None else parsed.replace(tzinfo=timezone.utc)
return parsed.replace(tzinfo=timezone.utc) if parsed is not None and parsed.tzinfo is None else parsed
def classify_pressure(available_kib: Any, total_kib: Any) -> str:
"""Map a MemAvailable/MemTotal pair to ``ok``/``elevated``/``critical``.
``unknown`` when the sample is missing or malformed — the caller must
not treat "we could not read it" as "memory is fine".
"""
available = _nonneg_int(available_kib)
"""``ok``/``elevated``/``critical`` from MemAvailable/MemTotal; ``unknown`` when the
sample is missing/malformed — "could not read it" must never read as "fine"."""
available, total = _nonneg_int(available_kib), _nonneg_int(total_kib)
if available is None:
return "unknown"
total = _nonneg_int(total_kib)
fraction = available / total if total else None
for level, kib_floor, frac_floor in (
("critical", _CRITICAL_AVAILABLE_KIB, _CRITICAL_AVAILABLE_FRACTION),
("elevated", _ELEVATED_AVAILABLE_KIB, _ELEVATED_AVAILABLE_FRACTION),
):
for level, kib_floor, frac_floor in _PRESSURE_TIERS:
if available < kib_floor or (fraction is not None and fraction < frac_floor):
return level
return "ok"
def _read_heartbeat(home: Optional[Path]) -> Optional[Dict[str, Any]]:
try:
from gateway.lifecycle_ledger import _read_json
from gateway.shutdown_watchdog import get_loop_heartbeat_path
return _read_json(get_loop_heartbeat_path(home))
except Exception:
return None
def _read_sentinel(home: Optional[Path]) -> Optional[Dict[str, Any]]:
def _read_state_files(home: Optional[Path]) -> tuple:
"""``(heartbeat, sentinel)`` dicts, each ``None`` when unreadable."""
try:
from gateway.lifecycle_ledger import _read_json, get_lifecycle_sentinel_path
from gateway.shutdown_watchdog import get_loop_heartbeat_path
return _read_json(get_lifecycle_sentinel_path(home))
return _read_json(get_loop_heartbeat_path(home)), _read_json(get_lifecycle_sentinel_path(home))
except Exception:
return None
return None, None
def collect_memory_status(
@@ -106,38 +76,25 @@ def collect_memory_status(
*,
now: Optional[datetime] = None,
) -> Dict[str, Any]:
"""Build the ``memory`` block for ``/api/status``.
``home`` scopes the read to a profile's HERMES_HOME (``None`` = active
profile); ``now`` is injectable for tests. Always returns a dict and never
raises — a down/never-started gateway or corrupt files yield
``{"pressure": "unknown", ...}`` plus whatever fields could be recovered.
"""
"""``memory`` block for ``/api/status``; ``home`` scopes to a profile (``None`` =
active), ``now`` is injectable. Never raises — a down gateway or corrupt files
yield ``pressure="unknown"`` plus whatever fields could be recovered."""
moment = now or datetime.now(timezone.utc)
status: Dict[str, Any] = {
"pressure": "unknown",
"gateway_rss_mb": None,
"system_total_mb": None,
"system_available_mb": None,
"swap_used_mb": None,
"sampled_at": None,
"last_boot_unclean": False,
"last_boot_suspected_oom": False,
# Identity of the CURRENT gateway life (sentinel started_at) — changes on
# every boot, so the dashboard can key banner dismissal on it and
# acknowledging one OOM restart does not mute the NEXT one.
"pressure": "unknown", "gateway_rss_mb": None, "system_total_mb": None, "system_available_mb": None,
"swap_used_mb": None, "sampled_at": None, "last_boot_unclean": False, "last_boot_suspected_oom": False,
# Identity of the CURRENT life (sentinel started_at): the dashboard keys
# banner dismissal on it so acknowledging one OOM restart does not mute the NEXT.
"boot_id": None,
}
heartbeat = _read_heartbeat(home)
heartbeat, sentinel = _read_state_files(home)
if heartbeat:
sampled_at = _parse_iso(heartbeat.get("updated_at"))
mem = heartbeat.get("mem")
sampled_at, mem = _parse_iso(heartbeat.get("updated_at")), heartbeat.get("mem")
if isinstance(mem, dict):
status["gateway_rss_mb"] = _mb(mem.get("rss_kib"))
status["system_total_mb"] = _mb(mem.get("mem_total_kib"))
status["system_available_mb"] = _mb(mem.get("mem_available_kib"))
status["swap_used_mb"] = _mb(mem.get("swap_used_kib"))
for dst, src in (("gateway_rss_mb", "rss_kib"), ("system_total_mb", "mem_total_kib"),
("system_available_mb", "mem_available_kib"), ("swap_used_mb", "swap_used_kib")):
status[dst] = _mb(mem.get(src))
if sampled_at is not None:
status["sampled_at"] = sampled_at.isoformat()
# Stale sample: numbers still reported (sampled_at says when) but
@@ -146,12 +103,10 @@ def collect_memory_status(
if 0 <= (moment - sampled_at).total_seconds() <= _HEARTBEAT_FRESH_TTL_S:
status["pressure"] = classify_pressure(mem.get("mem_available_kib"), mem.get("mem_total_kib"))
sentinel = _read_sentinel(home)
if sentinel:
status["last_boot_unclean"] = bool(sentinel.get("prior_unclean_exit"))
status["last_boot_suspected_oom"] = bool(sentinel.get("prior_suspected_oom"))
started_at = sentinel.get("started_at")
if isinstance(started_at, str) and started_at:
status["boot_id"] = started_at
status["boot_id"] = started_at if isinstance(started_at, str) and started_at else None
return status
gateway/message_timestamps.py
+36 -59
View File
@@ -12,17 +12,19 @@ from datetime import datetime
from typing import Any, Optional, Tuple
# Current gateway format: [Tue 2026-04-28 13:40:53 CEST]
_HUMAN_TIMESTAMP_RE = re.compile(
r"^\[(?P<dow>[A-Z][a-z]{2}) "
# Leading timestamp prefix, either the current human format
# ``[Tue 2026-04-28 13:40:53 CEST]`` or the older ISO one
# ``[2026-04-13T17:02:06+0200]`` / ``[...+02:00]`` (human tried first).
_TIMESTAMP_PREFIX_RE = re.compile(
r"^\[(?:"
r"(?P<dow>[A-Z][a-z]{2}) "
r"(?P<date>\d{4}-\d{2}-\d{2}) "
r"(?P<time>\d{2}:\d{2}:\d{2})"
r"(?: (?P<tz>[A-Za-z0-9_+\-/:]+))?\]\s*"
r"(?: (?P<tz>[A-Za-z0-9_+\-/:]+))?"
r"|(?P<iso>\d{4}-\d{2}-\d{2}T[^\]]+)"
r")\]\s*"
)
# Older gateway format: [2026-04-13T17:02:06+0200] or [+02:00]
_ISO_TIMESTAMP_RE = re.compile(r"^\[(?P<iso>\d{4}-\d{2}-\d{2}T[^\]]+)\]\s*")
def _localize(dt: datetime, tz) -> float:
"""Epoch for ``dt``; naive values take ``tz`` if given, else the local zone."""
@@ -33,18 +35,16 @@ def _localize(dt: datetime, tz) -> float:
def _parse_iso(text: str, tz=None) -> Optional[float]:
"""Parse an ISO-8601 string (incl. ``+0200`` offsets fromisoformat rejects)."""
try:
dt = datetime.fromisoformat(text)
except (TypeError, ValueError):
for parse in (datetime.fromisoformat, lambda t: datetime.strptime(t, "%Y-%m-%dT%H:%M:%S%z")):
try:
dt = datetime.strptime(text, "%Y-%m-%dT%H:%M:%S%z")
return _localize(parse(text), tz)
except (TypeError, ValueError):
return None
return _localize(dt, tz)
continue
return None
def _parse_timestamp_match(match: re.Match, tz=None) -> Optional[float]:
if match.groupdict().get("iso"):
if match.group("iso"):
return _parse_iso(match.group("iso"), tz)
try:
dt = datetime.strptime(f"{match.group('date')} {match.group('time')}", "%Y-%m-%d %H:%M:%S")
@@ -53,18 +53,9 @@ def _parse_timestamp_match(match: re.Match, tz=None) -> Optional[float]:
return _localize(dt, tz)
def _match_timestamp_prefix(text: str) -> Optional[re.Match]:
return _HUMAN_TIMESTAMP_RE.match(text) or _ISO_TIMESTAMP_RE.match(text)
def coerce_message_timestamp(ts_value: Any, tz=None) -> Optional[float]:
"""Coerce a timestamp-like value to Unix epoch seconds.
Accepts epoch numbers, datetime objects, ISO strings, and the gateway's
bracketed human-readable format. Returns ``None`` when uninterpretable.
"""
if ts_value is None:
return None
"""Epoch seconds from a number, datetime, ISO string, or the gateway's bracketed
format; ``None`` when uninterpretable."""
if isinstance(ts_value, (int, float)):
return float(ts_value)
if hasattr(ts_value, "timestamp"):
@@ -72,21 +63,17 @@ def coerce_message_timestamp(ts_value: Any, tz=None) -> Optional[float]:
return float(ts_value.timestamp())
except Exception:
return None
if isinstance(ts_value, str):
text = ts_value.strip()
if not text:
return None
match = _match_timestamp_prefix(text)
if match is not None:
parsed = _parse_timestamp_match(match, tz=tz)
if parsed is not None:
return parsed
try:
return float(text)
except (TypeError, ValueError):
pass
text = ts_value.strip() if isinstance(ts_value, str) else ""
if not text:
return None
match = _TIMESTAMP_PREFIX_RE.match(text)
parsed = _parse_timestamp_match(match, tz=tz) if match is not None else None
if parsed is not None:
return parsed
try:
return float(text)
except (TypeError, ValueError):
return _parse_iso(text, tz)
return None
def format_message_timestamp(ts_value: Any, tz=None) -> str:
@@ -95,21 +82,17 @@ def format_message_timestamp(ts_value: Any, tz=None) -> str:
if epoch is None:
return ""
dt = datetime.fromtimestamp(epoch, tz=tz) if tz is not None else datetime.fromtimestamp(epoch).astimezone()
return "[" + dt.strftime("%a %Y-%m-%d %H:%M:%S %Z") + "]"
return f"[{dt.strftime('%a %Y-%m-%d %H:%M:%S %Z')}]"
def strip_leading_message_timestamps(content: str, tz=None) -> Tuple[str, Optional[float]]:
"""Strip one or more leading gateway timestamp prefixes from ``content``.
Returns ``(clean_content, embedded_epoch)``. With multiple prefixes the one
closest to the message text wins, preserving the original platform-send
time for legacy contaminated rows like ``[processing time] [platform time] [sender] message``.
"""
"""Strip leading gateway timestamp prefixes → ``(clean_content, embedded_epoch)``.
With several prefixes the one closest to the text wins, preserving the platform-send
time of legacy rows like ``[processing time] [platform time] [sender] message``."""
if not isinstance(content, str) or not content:
return content, None
text = content
embedded_epoch: Optional[float] = None
while (match := _match_timestamp_prefix(text)) is not None:
text, embedded_epoch = content, None
while (match := _TIMESTAMP_PREFIX_RE.match(text)) is not None:
parsed = _parse_timestamp_match(match, tz=tz)
if parsed is not None:
embedded_epoch = parsed
@@ -118,14 +101,8 @@ def strip_leading_message_timestamps(content: str, tz=None) -> Tuple[str, Option
def render_user_content_with_timestamp(content: str, ts_value: Any = None, tz=None) -> str:
"""Render a user message for LLM context with exactly one timestamp prefix.
An existing leading prefix is stripped and its parsed time wins over
``ts_value``. If no timestamp is available the cleaned content is returned.
"""
"""Render a user message for LLM context with exactly one timestamp prefix; an
existing prefix is stripped and its parsed time wins over ``ts_value``."""
clean_content, embedded_epoch = strip_leading_message_timestamps(content, tz=tz)
effective_ts = embedded_epoch if embedded_epoch is not None else ts_value
prefix = format_message_timestamp(effective_ts, tz=tz)
if not prefix:
return clean_content
return f"{prefix} {clean_content}" if clean_content else prefix
prefix = format_message_timestamp(ts_value if embedded_epoch is None else embedded_epoch, tz=tz)
return f"{prefix} {clean_content}" if prefix and clean_content else (prefix or clean_content)
gateway/mirror.py
+41 -75
View File
@@ -1,10 +1,8 @@
"""
Session mirroring for cross-platform message delivery.
"""Session mirroring for cross-platform message delivery.
When a message is sent to a platform (send_message or cron delivery), append
a "delivery-mirror" record to the target session's transcript so the
receiving-side agent knows what was sent. Standalone: works from CLI, cron
and gateway contexts without the full SessionStore machinery.
When a message is sent to a platform (send_message or cron delivery), append a
"delivery-mirror" record to the target session's transcript so the receiving-side
agent knows what was sent. Standalone: works from CLI, cron and gateway contexts.
"""
import json
@@ -25,26 +23,19 @@ def _origin_user_id(entry: dict) -> str:
def mirror_to_session(
platform: str, chat_id: str, message_text: str, source_label: str = "cli",
thread_id: Optional[str] = None, user_id: Optional[str] = None,
role: str = "assistant", session_id: Optional[str] = None,
platform: str, chat_id: str, message_text: str, source_label: str = "cli", thread_id: Optional[str] = None,
user_id: Optional[str] = None, role: str = "assistant", session_id: Optional[str] = None,
) -> bool:
"""Append a delivery-mirror message to the target session's SQLite transcript.
``session_id``: pass it when the caller already holds the exact session
(e.g. the cron in_channel seed that just created the row) to skip the
origin scan. ``_find_session_id`` refuses to guess on a populated chat
(flat session + N thread sessions sharing one chat_id) and would silently
drop the mirror.
``role`` defaults to ``"assistant"``, correct when the mirrored text is
the agent's own outgoing reply. Text that is NOT the agent speaking (e.g.
a cron brief) must pass ``role="user"``: ``mirror``/``mirror_source``
metadata is dropped at the SQLite boundary, so an assistant-role mirror
replays as a real assistant turn and produces assistant→assistant pairs
that break strict-alternation providers; a user-role mirror collapses
safely via ``repair_message_sequence``'s consecutive-user merge.
Pass ``session_id`` when the caller already holds the exact session (e.g. the
cron in_channel seed) to skip the origin scan, which refuses to guess on a
populated chat (flat + N thread sessions per chat_id) and would drop the mirror.
Text that is NOT the agent speaking (e.g. a cron brief) must pass
``role="user"``: ``mirror`` metadata is dropped at the SQLite boundary, so an
assistant-role mirror replays as a real turn and yields assistant→assistant
pairs that break strict-alternation providers, while a user-role mirror
collapses safely via the consecutive-user merge.
Returns True if mirrored, False if no matching session or error. Never raises.
"""
try:
@@ -52,52 +43,38 @@ def mirror_to_session(
session_id = _find_session_id(platform, str(chat_id), thread_id=thread_id, user_id=user_id)
if not session_id:
logger.warning(
"Mirror: no session found for %s:%s thread=%s user=%s "
"(explicit_id=none, origin-scan bailed)",
"Mirror: no session found for %s:%s thread=%s user=%s (explicit_id=none, origin-scan bailed)",
platform, chat_id, thread_id, user_id,
)
return False
_append_to_sqlite(session_id, {
"role": role,
"content": message_text,
"timestamp": datetime.now().isoformat(),
"mirror": True,
"mirror_source": source_label,
"role": role, "content": message_text, "timestamp": datetime.now().isoformat(),
"mirror": True, "mirror_source": source_label,
})
logger.debug("Mirror: wrote to session %s (from %s)", session_id, source_label)
return True
except Exception as e:
# WARNING, not debug: a silent mirror drop is the cron continuation-amnesia bug.
logger.warning(
"Mirror failed for %s:%s thread=%s user=%s session=%s: %s",
platform, chat_id, thread_id, user_id, session_id, e,
)
logger.warning("Mirror failed for %s:%s thread=%s user=%s session=%s: %s", platform, chat_id, thread_id, user_id, session_id, e)
return False
def _find_session_id(
platform: str, chat_id: str, thread_id: Optional[str] = None, user_id: Optional[str] = None,
) -> Optional[str]:
"""Find the active session_id for a platform + chat_id pair.
def _find_session_id(platform: str, chat_id: str, thread_id: Optional[str] = None, user_id: Optional[str] = None) -> Optional[str]:
"""Active session_id for a platform + chat_id pair.
state.db gateway session rows are primary; sessions.json is the fallback
for pre-migration databases. DM session keys don't embed the chat_id
(e.g. "agent:main:telegram:dm"), so matching is on the persisted origin.
With *user_id*, exact sender matches win. If several same-chat candidates
exist and none matches the user, return None rather than guess and
contaminate another participant's session.
state.db is primary; sessions.json is the pre-migration fallback. DM keys
don't embed the chat_id ("agent:main:telegram:dm"), so match on the persisted
origin. With *user_id*, exact sender matches win; several same-chat candidates
with no user match → None rather than contaminate another participant's session.
"""
try:
from hermes_state import get_shared_session_db, release_or_close
db = get_shared_session_db()
try:
finder = getattr(db, "find_session_by_origin", None)
if callable(finder):
session_id = finder(platform=platform, chat_id=chat_id, thread_id=thread_id, user_id=user_id)
if session_id:
return str(session_id)
session_id = finder(platform=platform, chat_id=chat_id, thread_id=thread_id, user_id=user_id) if callable(finder) else None
if session_id:
return str(session_id)
finally:
release_or_close(db)
except Exception as e:
@@ -106,26 +83,18 @@ def _find_session_id(
if not _SESSIONS_INDEX.exists():
return None
try:
with open(_SESSIONS_INDEX, encoding="utf-8") as f:
data = json.load(f)
data = json.loads(_SESSIONS_INDEX.read_text(encoding="utf-8"))
except Exception:
return None
platform_lower = platform.lower()
candidates = []
for _key, entry in data.items():
# Keys starting with "_" (e.g. the gateway's "_README") are metadata sentinels.
if str(_key).startswith("_") or not isinstance(entry, dict):
continue
def _matches(entry: dict) -> bool:
origin = entry.get("origin") or {}
if (origin.get("platform") or entry.get("platform", "")).lower() != platform_lower:
continue
if str(origin.get("chat_id", "")) != str(chat_id):
continue
if thread_id is not None and str(origin.get("thread_id") or "") != str(thread_id):
continue
candidates.append(entry)
return ((origin.get("platform") or entry.get("platform", "")).lower() == platform.lower()
and str(origin.get("chat_id", "")) == str(chat_id)
and (thread_id is None or str(origin.get("thread_id") or "") == str(thread_id)))
# Keys starting with "_" (e.g. the gateway's "_README") are metadata sentinels.
candidates = [e for k, e in data.items() if not str(k).startswith("_") and isinstance(e, dict) and _matches(e)]
if not candidates:
return None
if user_id:
@@ -134,23 +103,20 @@ def _find_session_id(
candidates = exact_user_matches
elif len(candidates) > 1:
return None
elif len(candidates) > 1:
distinct_user_ids = {uid.strip() for uid in map(_origin_user_id, candidates) if uid.strip()}
if len(distinct_user_ids) > 1:
return None
elif len(candidates) > 1 and len({u.strip() for u in map(_origin_user_id, candidates) if u.strip()}) > 1:
return None
return max(candidates, key=lambda entry: entry.get("updated_at", "")).get("session_id")
def _append_to_sqlite(session_id: str, message: dict) -> None:
"""Append a message to the SQLite session database."""
db = None
try:
from hermes_state import get_shared_session_db, release_or_close
db = get_shared_session_db()
db.append_message(session_id=session_id, role=message.get("role", "assistant"), content=message.get("content"))
try:
db.append_message(session_id=session_id, role=message.get("role", "assistant"), content=message.get("content"))
finally:
release_or_close(db)
except Exception as e:
logger.debug("Mirror SQLite write failed: %s", e)
finally:
if db is not None:
release_or_close(db)
gateway/pairing.py
+151 -262
View File
@@ -1,8 +1,6 @@
"""
DM Pairing System
"""DM pairing: code-based approval of new users on messaging platforms.
Code-based approval flow for authorizing new users on messaging platforms:
unknown users receive a one-time pairing code that the bot owner approves via
Unknown users receive a one-time pairing code that the bot owner approves via
the CLI, instead of maintaining static user-ID allowlists.
Security properties (OWASP + NIST SP 800-63-4): 8-char codes from a 32-char
@@ -47,30 +45,23 @@ LOCKOUT_SECONDS = 3600 # Lockout duration after too many failures
MAX_PENDING_PER_PLATFORM = 3 # Max pending codes per platform
MAX_FAILED_ATTEMPTS = 5 # Failed approvals before lockout
# Default (non-profile-scoped) pairing directory. Deliberately ``None``: the
# long-lived gateway imports this module at boot, and an eagerly computed path
# would freeze whatever HERMES_HOME/profile context existed then, ignoring later
# context-local overrides (hermes_constants.set_hermes_home_override) -- so the
# gateway and the short-lived ``hermes pairing`` CLI ended up writing different
# directories. ``_default_pairing_dir()`` resolves fresh on every call; tests
# patch this attribute to a concrete path (``patch("gateway.pairing.PAIRING_DIR", tmp)``)
# and a non-``None`` value takes precedence.
# Default pairing directory override. Deliberately ``None``: an eagerly computed
# path would freeze the HERMES_HOME/profile context at gateway boot, ignoring later
# context-local overrides, so the gateway and ``hermes pairing`` CLI wrote different
# directories. ``_default_pairing_dir()`` resolves fresh per call; tests patch this.
PAIRING_DIR = None
def _default_pairing_dir() -> Path:
"""Resolve the default pairing directory fresh (see ``PAIRING_DIR``)."""
if PAIRING_DIR is not None:
return PAIRING_DIR
return get_hermes_dir("platforms/pairing", "pairing")
# Platform value -> per-platform allowlist env var. When an operator already
# runs an allowlist for a platform, approving a pairing code also writes the
# user into it (and revoking removes them), so the operator's list stays the
# single visible source of truth instead of drifting from an opaque
# approved.json. Platforms absent here (or with no allowlist configured) keep
# the pairing store as the sole grant record, honored by the authz union.
# Platform value -> allowlist env var. Approving a code also writes the user into
# an already-configured allowlist (revoke removes them) so the operator's list stays
# the visible source of truth. Platforms absent here (or with no allowlist
# configured) keep the pairing store as the sole grant record (authz union).
_PLATFORM_ALLOWLIST_ENV = {
"telegram": "TELEGRAM_ALLOWED_USERS",
"discord": "DISCORD_ALLOWED_USERS",
@@ -99,14 +90,10 @@ def _allowlist_env_for_platform(platform: str) -> Optional[str]:
env_var = _PLATFORM_ALLOWLIST_ENV.get(platform)
if env_var:
return env_var
try:
with contextlib.suppress(Exception):
from gateway.platform_registry import platform_registry
entry = platform_registry.get(platform)
if entry and entry.allowed_users_env:
return entry.allowed_users_env
except Exception:
pass
return platform_registry.get(platform).allowed_users_env or None
return None
@@ -128,11 +115,10 @@ def _normalize_user_id(platform: str, user_id: str) -> str:
def _user_id_aliases(platform: str, user_id: str) -> set[str]:
"""Return all known equivalent user IDs for auth / allowlist matching."""
"""All known equivalent user IDs for auth / allowlist matching."""
raw_user_id = str(user_id or "").strip()
if not raw_user_id:
return set()
aliases = {raw_user_id, _normalize_user_id(platform, raw_user_id)}
if _platform_uses_whatsapp_identity(platform):
aliases.update(expand_whatsapp_aliases(raw_user_id))
@@ -141,48 +127,50 @@ def _user_id_aliases(platform: str, user_id: str) -> set[str]:
def _user_ids_match(platform: str, left: str, right: str) -> bool:
"""Return True when two user IDs represent the same principal."""
"""True when two user IDs represent the same principal."""
left_aliases = _user_id_aliases(platform, left)
right_aliases = _user_id_aliases(platform, right)
return bool(left_aliases and right_aliases and (left_aliases & right_aliases))
return bool(left_aliases and left_aliases & _user_id_aliases(platform, right))
def _matching_ids(platform: str, approved: dict, user_id: str) -> list:
return [uid for uid in approved if _user_ids_match(platform, uid, user_id)]
def _read_allowlist_env(env_var: str) -> str:
"""Read a platform allowlist env var through the profile secret scope.
Under multiplexing the process env may hold ANOTHER profile's allowlist
(first-writer-wins YAML→env bridges), so a scoped miss must return empty
rather than borrow the process value. Unscoped callers (single-profile
CLI / admin endpoints) keep the legacy ``os.getenv`` read. Writes go
through ``hermes_cli.config.save_env_value`` / ``remove_env_value``, which
target the active profile's ``.env`` and, under multiplexing, publish into
the installed scope mapping rather than the shared ``os.environ``.
Under multiplexing the process env may hold ANOTHER profile's allowlist, so a
scoped miss must return empty rather than borrow it. Unscoped callers keep the
legacy ``os.getenv`` read. Writes go through ``save_env_value``/``remove_env_value``,
which target the active profile's ``.env`` / installed scope, not ``os.environ``.
"""
try:
with contextlib.suppress(Exception):
from agent.secret_scope import UnscopedSecretError, get_secret
try:
return (get_secret(env_var) or "").strip()
except UnscopedSecretError:
pass
except Exception:
pass
return (os.getenv(env_var) or "").strip()
def _configured_allowlist(platform: str):
"""Return ``(env_var, ids)`` for a platform whose allowlist is configured, else None.
"""``(env_var, ids)`` for a platform whose allowlist is configured, else None.
An unconfigured allowlist means an open gateway: the pairing store stays the
sole grant record and we must never lock the gateway by materializing one.
"""
env_var = _allowlist_env_for_platform(platform)
if not env_var:
return None
current = _read_allowlist_env(env_var)
if not current:
return None
return env_var, _split_allowlist(current)
current = _read_allowlist_env(env_var) if env_var else ""
return (env_var, _split_allowlist(current)) if current else None
def _write_allowlist_env(env_var: str, ids: list) -> None:
"""Best-effort persist (empty list removes the key); the pairing store grant still authorizes via the union."""
with contextlib.suppress(Exception):
from hermes_cli.config import save_env_value, remove_env_value
save_env_value(env_var, ",".join(ids)) if ids else remove_env_value(env_var)
def _sync_allowlist_add(platform: str, user_id: str) -> None:
@@ -192,31 +180,22 @@ def _sync_allowlist_add(platform: str, user_id: str) -> None:
return
env_var, ids = configured
if "*" in ids or str(user_id) in ids:
return # Already covered.
ids.append(str(user_id))
try:
from hermes_cli.config import save_env_value
save_env_value(env_var, ",".join(ids))
except Exception:
# Best-effort: the pairing store grant still authorizes via the union.
pass
return
_write_allowlist_env(env_var, [*ids, str(user_id)])
def _iter_live_gateway_adapters():
"""Yield adapters from the in-process GatewayRunner, if one is running."""
try:
runner = None
with contextlib.suppress(Exception):
from gateway.run import _gateway_runner_ref
runner = _gateway_runner_ref()
except Exception:
return
if runner is None:
return
for adapter in (getattr(runner, "adapters", None) or {}).values():
if adapter is not None:
yield adapter
for mapping in (getattr(runner, "_profile_adapters", None) or {}).values():
mappings = [getattr(runner, "adapters", None) or {}]
mappings.extend((getattr(runner, "_profile_adapters", None) or {}).values())
for mapping in mappings:
for adapter in (mapping or {}).values():
if adapter is not None:
yield adapter
@@ -232,25 +211,20 @@ def _adapter_platform_name(adapter) -> str:
def _purge_allowlist_entries(entries, platform: str, user_id: str):
"""Drop alias-equivalent allowlist entries while preserving ``*``."""
def keep(entry) -> bool:
entry = str(entry)
return entry.strip() == "*" or not _user_ids_match(platform, entry, str(user_id))
return str(entry).strip() == "*" or not _user_ids_match(platform, str(entry), str(user_id))
if isinstance(entries, str):
return ",".join(part for part in _split_allowlist(entries) if keep(part))
return ",".join(filter(keep, _split_allowlist(entries)))
if isinstance(entries, (set, frozenset)):
return {entry for entry in entries if keep(entry)}
return set(filter(keep, entries))
if isinstance(entries, (list, tuple)):
return [entry for entry in entries if keep(entry)]
return list(filter(keep, entries))
return entries
def _sync_live_adapter_allowlist_remove(platform: str, user_id: str) -> None:
"""Clear revoked principals from in-process adapter allowlist snapshots.
Adapters snapshot ``_allow_from`` at construction; when the revoked
principal was the sole entry the env key is removed entirely, so intake
must not keep authorizing from the stale snapshot until restart.
"""
"""Clear revoked principals from in-process adapter ``_allow_from`` snapshots,
so intake does not keep authorizing from a stale snapshot until restart."""
platform_name = (platform or "").strip().lower()
if not platform_name or not str(user_id or "").strip():
return
@@ -273,10 +247,8 @@ def _sync_live_adapter_allowlist_remove(platform: str, user_id: str) -> None:
def _sync_allowlist_remove(platform: str, user_id: str) -> None:
"""Remove ``user_id`` (and WhatsApp alias equivalents) from the allowlist.
Approve mirrors a normalized phone while revoke is often given a JID or
device-suffix form, so matching uses alias rules -- exact-string delete
would leave the sender authorized. Also clears in-process adapter
snapshots so sole-entry revocation works without a gateway restart.
Approve mirrors a normalized phone while revoke is often given a JID/device
form, so matching uses alias rules -- exact delete would leave the sender authorized.
"""
configured = _configured_allowlist(platform)
if configured is None:
@@ -285,63 +257,67 @@ def _sync_allowlist_remove(platform: str, user_id: str) -> None:
remaining = _purge_allowlist_entries(ids, platform, user_id)
if len(remaining) == len(ids):
return # Not present.
try:
from hermes_cli.config import save_env_value, remove_env_value
if remaining:
save_env_value(env_var, ",".join(remaining))
else:
remove_env_value(env_var)
except Exception:
pass
_write_allowlist_env(env_var, remaining)
_sync_live_adapter_allowlist_remove(platform, user_id)
def _load_json_file(path: Path) -> dict:
if path.exists():
"""Read a JSON object; {} when missing, malformed, unreadable, or not a dict.
PermissionError is logged loudly: a 0600 file owned by another uid (Docker:
``docker exec`` as root wrote it, the gosu-dropped gateway can't read it)
would otherwise silently leave the user unauthorized.
"""
if not path.exists():
return {}
try:
data = json.loads(path.read_text(encoding="utf-8"))
return data if isinstance(data, dict) else {}
except PermissionError as e:
try:
data = json.loads(path.read_text(encoding="utf-8"))
return data if isinstance(data, dict) else {}
except (json.JSONDecodeError, OSError):
return {}
return {}
st = path.stat()
owner_info = f"owner_uid={st.st_uid} mode={oct(st.st_mode)[-4:]}"
except OSError:
owner_info = "<stat failed>"
euid = os.geteuid() if hasattr(os, "geteuid") else "n/a" # no geteuid on Windows
logger.warning(
"Pairing file %s exists but is not readable as uid=%s (%s; %s). "
"If you ran `docker exec <container> hermes pairing approve ...` as root, "
"re-run with `docker exec -u hermes <container> ...` and "
"chown the existing file to the hermes user, or restart the "
"container so the entrypoint can fix ownership.",
path, euid, owner_info, e,
)
return {}
except (json.JSONDecodeError, OSError):
return {}
def _merge_pairing_dir(active_dir: Path, alternate_dir: Path) -> None:
def _save_json_file(path: Path, data: dict) -> None:
_secure_write(path, json.dumps(data, indent=2, ensure_ascii=False))
def _migrate_split_pairing_dirs(*, home: Optional[Path] = None, active: Optional[Path] = None) -> None:
"""Merge split legacy (``pairing``) / new (``platforms/pairing``) data into the active dir.
If both exist, approved users in the inactive location must not be
silently ignored (they would be asked for a fresh code).
If both exist, approved users in the inactive location must not be silently
ignored (they would be asked for a fresh code). Active data wins on key conflict.
"""
if not alternate_dir.exists() or active_dir.resolve() == alternate_dir.resolve():
return
active_dir.mkdir(parents=True, exist_ok=True)
for src in alternate_dir.glob("*.json"):
if not src.is_file():
continue
dest = active_dir / src.name
merged = _load_json_file(src)
if not merged:
continue
current = _load_json_file(dest)
before = dict(current)
# Active data wins on key conflict; otherwise union the inactive data.
merged.update(current)
if merged != before:
_secure_write(dest, json.dumps(merged, indent=2, ensure_ascii=False))
def _migrate_split_pairing_dirs(
*,
home: Optional[Path] = None,
active: Optional[Path] = None,
) -> None:
home = home or get_hermes_home()
old_dir = home / "pairing"
new_dir = home / "platforms" / "pairing"
active = active if active is not None else _default_pairing_dir()
alternate = new_dir if active.resolve() == old_dir.resolve() else old_dir
_merge_pairing_dir(active, alternate)
alternate = home / "platforms" / "pairing" if active.resolve() == old_dir.resolve() else old_dir
if not alternate.exists() or active.resolve() == alternate.resolve():
return
active.mkdir(parents=True, exist_ok=True)
for src in alternate.glob("*.json"):
merged = _load_json_file(src) if src.is_file() else {}
if not merged:
continue
current = _load_json_file(active / src.name)
merged.update(current)
if merged != current:
_save_json_file(active / src.name, merged)
def _secure_write(path: Path, data: str) -> None:
@@ -362,23 +338,20 @@ def _secure_write(path: Path, data: str) -> None:
raise
def _is_hashed_entry(entry) -> bool:
return isinstance(entry, dict) and "salt" in entry and "hash" in entry
class PairingStore:
"""
Manages pairing codes and approved user lists.
"""Pairing codes and approved user lists.
Data files per platform: ``{platform}-pending.json``,
``{platform}-approved.json``, plus shared ``_rate_limits.json``.
With ``profile="<name>"`` storage resolves from that profile's own
HERMES_HOME with the same legacy/consolidated layout rules as
``hermes -p <name> pairing ...``, so multiplex gateways and profile-scoped
CLI approvals share one whitelist. Without a profile, storage is the global
pairing directory for the current HERMES_HOME.
Files per platform: ``{platform}-pending.json``, ``{platform}-approved.json``,
plus shared ``_rate_limits.json``. With ``profile="<name>"`` storage resolves from
that profile's HERMES_HOME exactly as ``hermes -p <name> pairing ...`` does, so
multiplex gateways and profile-scoped CLI approvals share one whitelist.
"""
def __init__(self, profile: Optional[str] = None):
# Resolved lazily: tests use a temp HERMES_HOME and may construct the
# store before the env is set.
if profile:
root = get_default_hermes_root()
profile_home = root if profile == "default" else root / "profiles" / profile
@@ -387,11 +360,9 @@ class PairingStore:
profile_home = None
self._dir = _default_pairing_dir()
self._dir.mkdir(parents=True, exist_ok=True)
# Merge the alternate old/new layout so upgrades cannot split approvals;
# a profile store must resolve exactly as ``hermes -p <profile>`` does.
# Merge the alternate old/new layout so upgrades cannot split approvals.
_migrate_split_pairing_dirs(home=profile_home, active=self._dir)
# Protects all read-modify-write cycles: platform adapters run
# concurrently in threads sharing one PairingStore.
# Adapters run concurrently in threads sharing one store.
self._lock = threading.RLock()
self._profile = profile # for diagnostics / log lines
@@ -409,51 +380,20 @@ class PairingStore:
def _rate_limit_path(self) -> Path:
return self._dir / "_rate_limits.json"
def _load_json(self, path: Path) -> dict:
if path.exists():
try:
return json.loads(path.read_text(encoding="utf-8"))
except PermissionError as e:
# Loud warning: a 0600 file owned by another uid (Docker:
# `docker exec` as root writes it, the gosu-dropped gateway
# can't read it) would otherwise be swallowed by the OSError
# branch and silently leave the user unauthorized.
try:
st = path.stat()
owner_info = f"owner_uid={st.st_uid} mode={oct(st.st_mode)[-4:]}"
except OSError:
owner_info = "<stat failed>"
euid = os.geteuid() if hasattr(os, "geteuid") else "n/a" # no geteuid on Windows
logger.warning(
"Pairing file %s exists but is not readable as uid=%s (%s; %s). "
"If you ran `docker exec <container> hermes pairing approve ...` as root, "
"re-run with `docker exec -u hermes <container> ...` and "
"chown the existing file to the hermes user, or restart the "
"container so the entrypoint can fix ownership.",
path, euid, owner_info, e,
)
return {}
except (json.JSONDecodeError, OSError):
return {}
return {}
def _save_json(self, path: Path, data: dict) -> None:
_secure_write(path, json.dumps(data, indent=2, ensure_ascii=False))
_load_json = staticmethod(_load_json_file)
_save_json = staticmethod(_save_json_file)
# ----- Approved users -----
def is_approved(self, platform: str, user_id: str) -> bool:
"""Check if a user is approved (paired) on a platform."""
approved = self._load_json(self._approved_path(platform))
return any(_user_ids_match(platform, uid, user_id) for uid in approved)
return bool(_matching_ids(platform, self._load_json(self._approved_path(platform)), user_id))
def list_approved(self, platform: str = None) -> list:
"""List approved users, optionally filtered by platform."""
results = []
platforms = [platform] if platform else self._all_platforms("approved")
for p in platforms:
approved = self._load_json(self._approved_path(p))
for uid, info in approved.items():
for p in [platform] if platform else self._all_platforms("approved"):
for uid, info in self._load_json(self._approved_path(p)).items():
results.append({"platform": p, "user_id": uid, **info})
return results
@@ -461,11 +401,8 @@ class PairingStore:
"""Add a user to the approved list. Must be called under self._lock."""
approved = self._load_json(self._approved_path(platform))
normalized_user_id = _normalize_user_id(platform, user_id)
for approved_user_id in [
uid for uid in approved if _user_ids_match(platform, uid, normalized_user_id)
]:
for approved_user_id in _matching_ids(platform, approved, normalized_user_id):
del approved[approved_user_id]
approved[normalized_user_id] = {
"user_name": user_name,
"approved_at": time.time(),
@@ -479,9 +416,7 @@ class PairingStore:
path = self._approved_path(platform)
with self._lock:
approved = self._load_json(path)
matching_ids = [
uid for uid in approved if _user_ids_match(platform, uid, user_id)
]
matching_ids = _matching_ids(platform, approved, user_id)
if not matching_ids:
return False
for approved_user_id in matching_ids:
@@ -495,34 +430,22 @@ class PairingStore:
@staticmethod
def _hash_code(code: str, salt: bytes) -> str:
"""Hash a pairing code with the given salt using SHA-256."""
return hashlib.sha256(salt + code.encode("utf-8")).hexdigest()
def _finish_approval(
self, platform: str, pending: dict, matched_key: str, matched_entry: dict
) -> dict:
def _finish_approval(self, platform: str, pending: dict, matched_key: str, matched_entry: dict) -> dict:
"""Remove a pending request and approve its user. Must hold self._lock."""
del pending[matched_key]
self._save_json(self._pending_path(platform), pending)
# A successful approval proves the requester is legitimate, so the
# brute-force streak must not carry over: the counter persists in
# _rate_limits.json and would otherwise accumulate isolated typos
# until a single fresh typo trips a spurious lockout.
# A successful approval proves legitimacy, so the persisted brute-force streak
# must not carry over (isolated typos would accumulate into a spurious lockout).
self._reset_failed_attempts(platform)
self._approve_user(
platform, matched_entry["user_id"], matched_entry.get("user_name", "")
)
result = {"user_id": matched_entry["user_id"], "user_name": matched_entry.get("user_name", "")}
self._approve_user(platform, result["user_id"], result["user_name"])
return result
return {
"user_id": matched_entry["user_id"],
"user_name": matched_entry.get("user_name", ""),
}
def generate_code(
self, platform: str, user_id: str, user_name: str = ""
) -> Optional[str]:
def generate_code(self, platform: str, user_id: str, user_name: str = "") -> Optional[str]:
"""Generate a pairing code for a new user.
Returns None if the user is rate-limited, the platform hit
@@ -533,9 +456,7 @@ class PairingStore:
self._cleanup_expired(platform)
normalized_user_id = _normalize_user_id(platform, user_id)
if self._is_locked_out(platform):
return None
if self._is_rate_limited(platform, user_id):
if self._is_locked_out(platform) or self._is_rate_limited(platform, user_id):
return None
pending = self._load_json(self._pending_path(platform))
if len(pending) >= MAX_PENDING_PER_PLATFORM:
@@ -556,30 +477,24 @@ class PairingStore:
return code
def approve_code(self, platform: str, code: str) -> Optional[dict]:
"""Approve a pairing code. Adds the user to the approved list.
"""Approve a pairing code and add its user to the approved list.
Returns ``{user_id, user_name}`` on success, ``None`` if the code is
invalid/expired OR the platform is locked out after
``MAX_FAILED_ATTEMPTS`` failed approvals (disambiguate with
``_is_locked_out``). The code is hashed with each entry's salt and
compared in constant time; legacy plaintext-key entries are ignored
and pruned at TTL by ``_cleanup_expired``.
Returns ``{user_id, user_name}``, or ``None`` if the code is invalid/expired OR
the platform is locked out (disambiguate with ``_is_locked_out``). Constant-time
salted-hash compare; legacy plaintext entries are ignored and pruned at TTL.
"""
with self._lock:
self._cleanup_expired(platform)
code = code.upper().strip()
# Must run before the pending lookup, or an already-issued valid
# code would still be accepted during lockout, nullifying the
# brute-force protection.
# Before the lookup, or an already-issued valid code would bypass lockout.
if self._is_locked_out(platform):
return None
pending = self._load_json(self._pending_path(platform))
# Skip legacy/malformed entries rather than KeyError so an in-place
# upgrade over an existing pending.json doesn't crash.
# Skip legacy/malformed entries so an in-place upgrade doesn't crash.
for entry_id, entry in pending.items():
if not isinstance(entry, dict) or "salt" not in entry or "hash" not in entry:
if not _is_hashed_entry(entry):
continue
try:
salt = bytes.fromhex(entry["salt"])
@@ -593,27 +508,18 @@ class PairingStore:
@staticmethod
def looks_like_request_id(value: str) -> bool:
"""True when ``value`` has the shape of a ``list_pending`` request id.
Request ids are ``secrets.token_hex(8)`` (16 hex chars); pairing codes
are 8 chars from an uppercase alphabet, so the shapes cannot collide
and callers accepting either can dispatch on this.
"""
"""True when ``value`` is shaped like a ``list_pending`` request id (16 hex chars);
pairing codes are 8 uppercase chars, so callers accepting either can dispatch on this."""
value = str(value or "").strip()
return len(value) == 16 and all(c in "0123456789abcdefABCDEF" for c in value)
def approve_request(self, platform: str, request_id: str) -> Optional[dict]:
"""Approve a pending request by its server-side request id.
"""Approve a pending request by its server-side request id (admin surfaces that
must never reveal the DM'd code). Returns ``{user_id, user_name}`` or ``None``.
Grant path for authenticated admin surfaces (``hermes pairing list``,
dashboard/desktop buttons) that must never reveal the DM'd code.
Returns ``{user_id, user_name}`` or ``None`` for an unknown/expired id.
Unlike :meth:`approve_code` this neither counts toward nor is gated by
the brute-force lockout: a request id is only obtainable by an admin
already authenticated to this store, so a stale id means "the row you
clicked expired", not an attack -- and counting it would let a few GUI
clicks lock the operator out of the CLI's code path.
Neither counts toward nor is gated by the brute-force lockout: a request id is only
obtainable by an authenticated admin, so a stale id is "the row expired", not an
attack -- counting it would let a few GUI clicks lock the operator out.
"""
with self._lock:
self._cleanup_expired(platform)
@@ -623,35 +529,24 @@ class PairingStore:
pending = self._load_json(self._pending_path(platform))
for entry_id, entry in pending.items():
if not isinstance(entry, dict) or "salt" not in entry or "hash" not in entry:
continue
if secrets.compare_digest(str(entry_id).lower(), request_id):
if _is_hashed_entry(entry) and secrets.compare_digest(str(entry_id).lower(), request_id):
return self._finish_approval(platform, pending, entry_id, entry)
return None
def list_pending(self, platform: str = None) -> list:
"""List pending pairing requests, optionally filtered by platform.
Codes are hashed and never returned; each entry exposes a server-side
``request_id`` for :meth:`approve_request`. Legacy pre-hash entries
report an empty ``request_id`` and age out at TTL.
"""
"""List pending requests (codes are never returned; each exposes a ``request_id``
for :meth:`approve_request`; legacy pre-hash entries report an empty id)."""
results = []
with self._lock:
platforms = [platform] if platform else self._all_platforms("pending")
for p in platforms:
for p in [platform] if platform else self._all_platforms("pending"):
self._cleanup_expired(p)
pending = self._load_json(self._pending_path(p))
for entry_id, info in pending.items():
for entry_id, info in self._load_json(self._pending_path(p)).items():
if not isinstance(info, dict):
continue
created_at = info.get("created_at")
if not isinstance(created_at, (int, float)):
continue
is_modern = isinstance(info.get("hash"), str) and isinstance(
info.get("salt"), str
)
is_modern = isinstance(info.get("hash"), str) and isinstance(info.get("salt"), str)
results.append({
"platform": p,
"request_id": str(entry_id) if is_modern else "",
@@ -665,17 +560,15 @@ class PairingStore:
"""Clear all pending requests. Returns count removed."""
with self._lock:
count = 0
platforms = [platform] if platform else self._all_platforms("pending")
for p in platforms:
pending = self._load_json(self._pending_path(p))
count += len(pending)
for p in [platform] if platform else self._all_platforms("pending"):
count += len(self._load_json(self._pending_path(p)))
self._save_json(self._pending_path(p), {})
return count
# ----- Rate limiting and lockout -----
def _is_rate_limited(self, platform: str, user_id: str) -> bool:
"""Check if a user (under any alias) has requested a code too recently."""
"""Whether a user (under any alias) has requested a code too recently."""
limits = self._load_json(self._rate_limit_path())
return any(
(time.time() - limits.get(f"{platform}:{alias}", 0)) < RATE_LIMIT_SECONDS
@@ -683,7 +576,6 @@ class PairingStore:
)
def _record_rate_limit(self, platform: str, user_id: str) -> None:
"""Record the time of a pairing request for rate limiting."""
limits = self._load_json(self._rate_limit_path())
now = time.time()
for alias in _user_id_aliases(platform, user_id):
@@ -691,19 +583,18 @@ class PairingStore:
self._save_json(self._rate_limit_path(), limits)
def _is_locked_out(self, platform: str) -> bool:
"""Check if a platform is in lockout due to failed approval attempts."""
limits = self._load_json(self._rate_limit_path())
return time.time() < limits.get(f"_lockout:{platform}", 0)
def _record_failed_attempt(self, platform: str) -> None:
"""Record a failed approval attempt. Triggers lockout after MAX_FAILED_ATTEMPTS."""
"""Record a failed approval attempt; triggers lockout after MAX_FAILED_ATTEMPTS."""
limits = self._load_json(self._rate_limit_path())
fail_key = f"_failures:{platform}"
fails = limits.get(fail_key, 0) + 1
limits[fail_key] = fails
if fails >= MAX_FAILED_ATTEMPTS:
limits[f"_lockout:{platform}"] = time.time() + LOCKOUT_SECONDS
limits[fail_key] = 0 # Reset counter
limits[fail_key] = 0
print(f"[pairing] Platform {platform} locked out for {LOCKOUT_SECONDS}s "
f"after {MAX_FAILED_ATTEMPTS} failed attempts", flush=True)
self._save_json(self._rate_limit_path(), limits)
@@ -730,12 +621,10 @@ class PairingStore:
or (now - info["created_at"]) > CODE_TTL_SECONDS
]
if expired:
for entry_id in expired:
del pending[entry_id]
self._save_json(path, pending)
self._save_json(path, {k: v for k, v in pending.items() if k not in expired})
def _all_platforms(self, suffix: str) -> list:
"""List all platforms that have data files of a given suffix."""
"""Platforms that have a ``-<suffix>.json`` data file (``_``-prefixed files are shared state)."""
tail = f"-{suffix}.json"
platforms = [f.name.replace(tail, "") for f in self._dir.iterdir() if f.name.endswith(tail)]
return [p for p in platforms if not p.startswith("_")]
gateway/platform_registry.py
+68 -168
View File
@@ -1,25 +1,11 @@
"""
Platform Adapter Registry
"""Platform adapter registry.
Platform adapters (built-in and plugin) self-register here so the gateway can
discover and instantiate them without hardcoded if/elif chains. Plugins
discover and instantiate them without hardcoded if/elif chains. Plugins
register via ``PluginContext.register_platform()``; ``GatewayRunner
._create_adapter()`` consults the registry first and falls back to the legacy
built-in path.
Usage (plugin side)::
platform_registry.register(PlatformEntry(
name="irc", label="IRC",
adapter_factory=lambda cfg: IRCAdapter(cfg),
check_fn=check_requirements,
validate_config=lambda cfg: bool(cfg.extra.get("server")),
required_env=["IRC_SERVER"], install_hint="pip install irc",
))
Usage (gateway side)::
adapter = platform_registry.create_adapter("irc", platform_config)
built-in path. Plugin side: ``platform_registry.register(PlatformEntry(...))``;
gateway side: ``platform_registry.create_adapter("irc", platform_config)``.
"""
import logging
@@ -60,113 +46,53 @@ def _caller_plugin_scope() -> Optional[str]:
class PlatformEntry:
"""Metadata and factory for a single platform adapter."""
# Identifier used in config.yaml (e.g. "irc") and human-readable label.
name: str
label: str
# Receives a PlatformConfig, returns an adapter instance. A factory (not a
# bare class) lets plugins do custom init / wrapping.
adapter_factory: Callable[[Any], Any]
# PASSIVE dependency probe: True when deps are importable RIGHT NOW. Must
# be side-effect free -- it runs from status displays (`hermes setup`,
# `hermes status`, dashboard readiness) and the config enablement pass,
# none of which may pip-install. Install logic belongs in ensure_deps_fn.
name: str # config.yaml identifier (e.g. "irc")
label: str # human-readable
adapter_factory: Callable[[Any], Any] # PlatformConfig -> adapter (factory allows custom init)
# PASSIVE dependency probe (deps importable RIGHT NOW). Must be side-effect free: it
# runs from status displays and the config enablement pass, which may never pip-install.
check_fn: Callable[[], bool]
# Optional config check. None = skip and let connect() fail descriptively.
validate_config: Optional[Callable[[Any], bool]] = None
# ACTIVE dependency installer: make deps importable (pip / lazy_deps),
# returning True on success. Called by ``create_adapter()`` only when
# ``check_fn`` is False -- the one moment the user has the platform
# enabled+configured and the gateway is about to connect it. None = a
# False ``check_fn`` is a hard block (correct for platforms without
# optional deps). The passive/active split exists because a single field
# either pip-installed from every status display or never installed at all.
validate_config: Optional[Callable[[Any], bool]] = None # None = let connect() fail descriptively
# ACTIVE installer, run by ``create_adapter()`` only when ``check_fn`` is False (the
# platform is enabled+configured and about to connect). None = a False check_fn is a
# hard block. The split exists because one field either installed from every status
# display or never at all.
ensure_deps_fn: Optional[Callable[[], bool]] = None
# Is the platform connected/enabled for this PlatformConfig? Used by
# ``GatewayConfig.get_connected_platforms()`` and setup UI status; None
# falls back to ``validate_config`` or ``check_fn``.
# Connected/configured for this PlatformConfig (``get_connected_platforms``, setup UI);
# None falls back to ``validate_config`` or ``check_fn``.
is_connected: Optional[Callable[[Any], bool]] = None
# Env vars this platform needs (``hermes setup`` display) and the hint
# shown when check_fn returns False.
required_env: list = field(default_factory=list)
install_hint: str = ""
# Interactive setup ``() -> None``. None falls back to
# _setup_standard_platform (needs token_var + vars) or a generic env display.
setup_fn: Optional[Callable[[], None]] = None
# "builtin" or "plugin"; plugin_name is the owning manifest (empty for
# built-ins) so ``hermes gateway setup`` can auto-enable the plugin.
source: str = "plugin"
plugin_name: str = ""
# Auth env var names for _is_user_authorized: comma-separated allowed user
# IDs, and a truthy "allow everyone" switch.
allowed_users_env: str = ""
allow_all_env: str = ""
# Max message length for smart-chunking; 0 = no limit.
max_message_length: int = 0
# If True, session descriptions redact PII (phone numbers, etc.).
pii_safe: bool = False
# Emoji for CLI/gateway display.
emoji: str = "🔌"
# Whether /update may be issued from this platform (_UPDATE_ALLOWED_PLATFORMS).
allow_update_command: bool = True
# Platform hint injected into the system prompt; empty = none.
platform_hint: str = ""
# Env-driven auto-configuration ``() -> Optional[dict]``: returns
# ``PlatformConfig.extra`` fields to seed when the platform is auto-enabled.
# Runs during ``_apply_env_overrides`` BEFORE the adapter is constructed so
# ``gateway status`` reflects env-only config. None/empty dict = skip.
required_env: list = field(default_factory=list) # ``hermes setup`` display
install_hint: str = "" # shown when check_fn is False
setup_fn: Optional[Callable[[], None]] = None # None = _setup_standard_platform / env display
source: str = "plugin" # "builtin" or "plugin"
plugin_name: str = "" # owning manifest so ``hermes gateway setup`` can auto-enable it
allowed_users_env: str = "" # comma-separated allowed user IDs (_is_user_authorized)
allow_all_env: str = "" # truthy "allow everyone" switch
max_message_length: int = 0 # smart-chunking cap; 0 = no limit
pii_safe: bool = False # session descriptions redact PII (phone numbers, etc.)
emoji: str = "🔌" # CLI/gateway display
allow_update_command: bool = True # /update may be issued from this platform
platform_hint: str = "" # injected into the system prompt; empty = none
# ``() -> Optional[dict]`` of ``extra`` fields to seed when auto-enabled from env. Runs
# in ``_apply_env_overrides`` BEFORE adapter construction so ``gateway status`` sees it.
env_enablement_fn: Optional[Callable[[], Optional[dict]]] = None
# YAML->env bridge ``(yaml_cfg, platform_cfg) -> Optional[dict]``: lets a
# plugin own its config.yaml translation instead of core gateway/config.py.
# Called from ``load_gateway_config()`` after the generic shared-key loop and
# before ``_apply_env_overrides``. May mutate ``os.environ`` (guard with
# ``not os.getenv(...)`` to keep env > YAML precedence); the returned dict is
# merged into ``PlatformConfig.extra``. Exceptions are logged at debug.
# Full contract: website/docs/developer-guide/adding-platform-adapters.md.
# YAML->env bridge ``(yaml_cfg, platform_cfg) -> Optional[dict]`` merged into ``extra``;
# runs after the shared-key loop, before ``_apply_env_overrides``. May set ``os.environ``
# (guard with ``not os.getenv(...)`` to keep env > YAML). Contract:
# website/docs/developer-guide/adding-platform-adapters.md.
apply_yaml_config_fn: Optional[Callable[[dict, dict], Optional[dict]]] = None
# Home-channel env var (e.g. "IRC_HOME_CHANNEL"). When set, cron.scheduler
# accepts ``deliver=<name>`` and reads it for the default chat/room ID.
cron_deliver_env_var: str = ""
# Target parsing ``(target_ref) -> Optional[(chat_id, thread_id)]``, run by
# ``tools/send_message_tool._parse_target_ref`` before channel-directory
# fallback so plugins can declare native target syntax
# (e.g. ``fmsg:@alice@example.com``). None result = continue to directory
# resolution; no opaque fallback is applied.
cron_deliver_env_var: str = "" # home-channel env var read for cron ``deliver=<name>``
# ``(target_ref) -> Optional[(chat_id, thread_id)]`` run before channel-directory
# fallback so plugins can declare native target syntax; None = continue resolution.
parse_target_ref_fn: Optional[Callable[[str], Optional[tuple[str, Optional[str]]]]] = None
# Validation after parsing/normalization or directory resolution: True to
# accept, False to reject, or a non-empty string to reject with diagnostic.
# Post-resolution validation: True accept, False reject, non-empty str = reject + diagnostic.
validate_target_ref_fn: Optional[Callable[[str], bool | str]] = None
# Whole-request delivery handler ``(args, normalized_chat_id, platform_name,
# pconfig)``, sync or async. Prefer standalone_sender_fn when the standard
# send contract suffices.
# Whole-request delivery ``(args, normalized_chat_id, platform_name, pconfig)``, sync or
# async. Prefer standalone_sender_fn when the standard send contract suffices.
send_message_handler: Optional[Callable[[dict, str, str, Any], Any]] = None
# Out-of-process sender used by ``_send_via_adapter`` when cron runs apart
# from the gateway and the in-process adapter weakref is None::
# async (pconfig, chat_id, message, *, thread_id=None,
# media_files=None, force_document=False) -> dict
# Returns ``{"success": True, "message_id": ...}`` or ``{"error": str}``.
# Without it, plugin platforms cannot be cron ``deliver=`` targets when the
# gateway is not co-resident.
# Out-of-process sender for cron without a co-resident gateway::
# async (pconfig, chat_id, message, *, thread_id=None, media_files=None, force_document=False) -> dict
# returning ``{"success": True, "message_id": ...}`` or ``{"error": str}``.
standalone_sender_fn: Optional[Callable[..., Awaitable[dict]]] = None
@@ -184,11 +110,9 @@ class PlatformRegistry:
# Plugin adapters are isolated per resolved HERMES_HOME and overlay the
# process-global entries for lookups in that profile's runtime scope.
self._scoped_entries: dict[str, dict[str, PlatformEntry]] = {}
# Deferred loaders: name -> zero-arg callable that imports the owning
# plugin module (which calls register()). Adapter modules import heavy
# SDKs at module level; eagerly loading ~20 bundled platforms added
# seconds to every `hermes` invocation, so the real import happens only
# when a lookup actually asks for that platform.
# Deferred loaders: name -> callable importing the owning plugin module (which calls
# register()). Eagerly importing ~20 SDK-heavy adapters added seconds to every
# `hermes` invocation, so the import happens only when a lookup asks for it.
self._deferred: dict[str, _Loader] = {}
self._scoped_deferred: dict[str, dict[str, _Loader]] = {}
self._inflight: dict[_LoadKey, threading.Event] = {}
@@ -220,29 +144,21 @@ class PlatformRegistry:
entry = entries.get(name)
loader = deferred.get(name)
if entry is None and loader is None:
loader = self._inflight_loaders.get((scope, name))
if entry is None and loader is None:
loader = self._consumed_loaders.get((scope, name))
loader = self._inflight_loaders.get((scope, name)) or self._consumed_loaders.get((scope, name))
return entry, loader
def _prune_scope(self, scope: Optional[str]) -> None:
if scope is None:
return
if not self._scoped_entries.get(scope):
self._scoped_entries.pop(scope, None)
if not self._scoped_deferred.get(scope):
self._scoped_deferred.pop(scope, None)
for maps in (self._scoped_entries, self._scoped_deferred):
if not maps.get(scope):
maps.pop(scope, None)
# -- deferred loading ----------------------------------------------------
def register_deferred(self, name: str, loader: _Loader, *, scope: Optional[str] = None) -> None:
"""Register a lazy loader for a platform that hasn't been imported yet.
*loader* imports the owning plugin module, which must call
:meth:`register` for *name*. It runs at most once, on first lookup
(or full materialization). A concrete registration takes precedence
and drops the loader.
"""
"""Register a lazy loader (imports the plugin module, which must call :meth:`register`).
Runs at most once, on first lookup; a concrete registration drops it."""
with self._lock:
entries, deferred = self._scope_maps(scope, create=True)
self._consumed_loaders.pop((scope, name), None)
@@ -268,12 +184,8 @@ class PlatformRegistry:
*,
scope: Optional[str] = None,
) -> bool:
"""Restore a registration if its full state is still *current* (CAS).
Identity checks protect a later registration from removal while letting
an unloaded override reveal what it displaced. Deferred loaders are
part of the state because bundled platform plugins load lazily.
"""
"""Restore a registration if its full state is still *current* (CAS): a later
registration is never removed, and deferred loaders are part of the state."""
with self._lock:
entry, loader = self._registration_state(scope, name, create=True)
if entry is not current[0] or loader is not current[1]:
@@ -355,18 +267,13 @@ class PlatformRegistry:
self._resolve(name, active_scope)
def is_deferred_load_cancelled(self, name: str, *, scope: Optional[str] = None) -> bool:
"""Return whether ownership teardown cancelled an in-flight loader."""
"""Whether ownership teardown cancelled an in-flight loader."""
with self._lock:
return (scope, name) in self._cancelled_inflight
def _resolve_all(self) -> None:
"""Run every pending deferred loader.
Only the iterate-all accessors (``all_entries``/``plugin_entries``) call
this, from paths that genuinely need every adapter (gateway startup,
``hermes setup``/``gateway status``, channel directory). CLI chat never
iterates the full set.
"""
"""Run every pending deferred loader (only ``all_entries``/``plugin_entries`` call
this, from paths that need every adapter; CLI chat never iterates the full set)."""
active_scope = self.current_scope_key()
with self._lock:
_entries, scoped_deferred = self._scope_maps(active_scope)
@@ -386,11 +293,11 @@ class PlatformRegistry:
"""Register a platform adapter entry (last writer wins on name clash)."""
with self._lock:
if scope is None and entry.source == "plugin":
scope = _caller_plugin_scope()
if scope is None:
scope = _plugin_scope_from_callable(entry.adapter_factory)
if scope is None:
scope = _plugin_scope_from_callable(entry.check_fn)
scope = (
_caller_plugin_scope()
or _plugin_scope_from_callable(entry.adapter_factory)
or _plugin_scope_from_callable(entry.check_fn)
)
# A concrete registration supersedes any pending deferred loader.
entries, deferred = self._scope_maps(scope, create=True)
self._consumed_loaders.pop((scope, entry.name), None)
@@ -404,7 +311,7 @@ class PlatformRegistry:
logger.debug("Registered platform adapter: %s (%s)", entry.name, entry.source)
def unregister(self, name: str, *, scope: Optional[str] = None) -> bool:
"""Remove a platform entry. Returns True if it existed."""
"""Remove a platform entry. Returns True if it existed."""
with self._lock:
inferred_scope = scope if scope is not None else _caller_plugin_scope()
active_scope = inferred_scope or self.current_scope_key()
@@ -452,12 +359,8 @@ class PlatformRegistry:
return [e for e in self.all_entries() if e.source == "plugin"]
def registered_names(self) -> set[str]:
"""Concrete and deferred platform names without loading adapters.
Same scope semantics as ``is_registered()``: current profile scope AND
process-global names. Plugin platforms register deferred loaders under
a profile scope, so the global maps alone would miss every plugin.
"""
"""Concrete and deferred names (current profile scope AND process-global, like
``is_registered``) without loading adapters."""
with self._lock:
entries, deferred = self._scope_maps(self.current_scope_key())
return entries.keys() | deferred.keys() | self._entries.keys() | self._deferred.keys()
@@ -481,17 +384,14 @@ class PlatformRegistry:
if entry is None:
return None
deps_ok = False
try:
deps_ok = bool(entry.check_fn())
except Exception as e:
logger.warning("Platform '%s' check_fn raised: %s", entry.label, e)
deps_ok = False
if not deps_ok and entry.ensure_deps_fn is not None:
# The ONE place the active installer runs in the adapter path: the
# platform is enabled+configured and about to connect, so an install
# is what the user wants. (An installer inside connect() is never
# reached when check_fn is False -- create_adapter() would have
# already returned None.)
# The ONE place the active installer runs: the platform is enabled+configured
# and about to connect, so an install is what the user wants.
logger.info("Platform '%s' dependencies missing — attempting install...", entry.label)
try:
deps_ok = bool(entry.ensure_deps_fn())
gateway/profile_routing.py
+21 -54
View File
@@ -1,35 +1,9 @@
"""Profile-based routing for the gateway with hierarchical matching.
"""Profile-based routing: route guilds/channels/threads to different profiles.
Lets a single Hermes instance route specific Discord guilds/channels/threads
to different profiles — each with their own model, tools, memory, and persona.
Matching priority (most specific first):
1. platform + chat_id + thread_id (exact thread) — specificity 14
2. platform + chat_id (channel route) — specificity 6
3. platform + guild_id (guild/server route) — specificity 2
4. No match → default profile
Parent-chain matching: for Discord threads and forum posts, ``parent_chat_id``
carries the direct parent, so a route keyed on a channel also matches messages
in any thread/post under it.
Configuration (config.yaml)::
gateway:
profile_routes:
- name: server-default
platform: discord
guild_id: "YOUR_GUILD_ID"
profile: server-profile
- name: thread-route
platform: discord
chat_id: "YOUR_CHANNEL_ID"
thread_id: "YOUR_THREAD_ID"
profile: thread-profile
- name: owner-whatsapp
platform: whatsapp
chat_id: "15551234567" # phone, JID, or LID — all equivalent
profile: owner
Matching priority, most specific first (``gateway.profile_routes`` in config.yaml):
platform + chat_id + thread_id (14) → platform + chat_id (6) → platform + guild_id (2)
→ default profile. For Discord threads/forum posts ``parent_chat_id`` carries the
direct parent, so a channel route also matches any thread/post under it.
"""
from __future__ import annotations
@@ -41,8 +15,7 @@ import logging
logger = logging.getLogger(__name__)
# Baileys and Cloud share phone/JID/LID identity rules. Other platforms keep
# exact string compare so Telegram numeric ids and Discord snowflakes stay unchanged.
# Baileys and Cloud share phone/JID/LID identity rules; other platforms compare exactly.
_WHATSAPP_IDENTITY_PLATFORMS = {"whatsapp", "whatsapp_cloud"}
_WHATSAPP_NON_USER_SUFFIXES = ("@g.us", "@broadcast", "@newsletter")
@@ -59,15 +32,14 @@ def _whatsapp_user_chat_ids_match(platform: str, left: Optional[str], right: Opt
"""True when two WhatsApp *user* chat_ids refer to the same person.
Uses ``expand_whatsapp_aliases`` (same helper as session keys and adapter
allowlists) so a bare number, ``@s.whatsapp.net`` JID and ``@lid`` LID
collapse to one identity. Group/broadcast JIDs are chats, not senders,
and non-WhatsApp platforms always return False (exact match only).
allowlists) so a bare number, JID and LID collapse to one identity.
Group/broadcast JIDs are chats, not senders; non-WhatsApp platforms → False.
"""
if (platform or "").strip().lower() not in _WHATSAPP_IDENTITY_PLATFORMS:
return False
if not left or not right:
return False
if _is_whatsapp_non_user_chat(left) or _is_whatsapp_non_user_chat(right):
if (
(platform or "").strip().lower() not in _WHATSAPP_IDENTITY_PLATFORMS
or not left or not right
or _is_whatsapp_non_user_chat(left) or _is_whatsapp_non_user_chat(right)
):
return False
from gateway.whatsapp_identity import expand_whatsapp_aliases
@@ -104,13 +76,11 @@ class ProfileRoute:
thread_id: Optional[str] = None,
parent_chat_id: Optional[str] = None,
) -> bool:
"""Return True if every discriminator the route declares holds (AND).
"""True if every discriminator the route declares holds (AND).
``chat_id`` matches the channel directly or as the parent of a
thread/forum post; a route with both ``guild_id`` and ``chat_id``
requires both. WhatsApp ``chat_id`` also matches across user-identity
forms (number/JID/LID) after the exact check; groups/broadcasts stay
exact-only.
``chat_id`` matches the channel directly or as the parent of a thread/forum
post; WhatsApp ``chat_id`` also matches across number/JID/LID after the
exact check (groups/broadcasts stay exact-only).
"""
if not self.enabled or self.platform != platform:
return False
@@ -129,11 +99,9 @@ class ProfileRoute:
def _coerce_route_id(value: Any) -> Optional[str]:
"""Normalize a route discriminator to str for strict equality matching.
PyYAML loads unquoted numeric IDs as ``int`` while inbound ``SessionSource``
fields are always ``str``, so ints must be coerced or ``matches()`` fails
silently. Only ``int`` (not ``bool``) is coerced; floats and other types
stringify to something (``"123.0"``) that can never match, so they pass
through with a load-time warning instead of being silently "fixed".
PyYAML loads unquoted numeric IDs as ``int`` while ``SessionSource`` fields are
``str``. Only ``int`` (not ``bool``) is coerced; floats stringify to something
(``"123.0"``) that can never match, so they get a load-time warning instead.
"""
if value is None or isinstance(value, str):
return value
@@ -161,8 +129,7 @@ def parse_profile_routes(raw: Optional[List[Dict[str, Any]]]) -> List[ProfileRou
if not platform or not profile:
logger.warning("Skipping profile route %s: missing platform or profile", name)
continue
# Validate profile name to prevent path traversal. Lazy import avoids a
# circular dependency at module load time.
# Validate profile name to prevent path traversal (lazy import: cycle).
try:
from hermes_cli.profiles import normalize_profile_name, validate_profile_name
gateway/readiness.py
+23 -45
View File
@@ -14,14 +14,11 @@ from hermes_constants import get_hermes_home
_DISK_DEGRADED_PERCENT = 90.0
_CONNECTED_STATES = {"connected", "running", "ok"}
def _check(status: str, detail: str | None = None, **extra: Any) -> dict[str, Any]:
result: dict[str, Any] = {"status": status}
if detail:
result["detail"] = detail
result.update(extra)
return result
return {"status": status, **({"detail": detail} if detail else {}), **extra}
def _probe_state_db(home: Path) -> dict[str, Any]:
@@ -29,12 +26,10 @@ def _probe_state_db(home: Path) -> dict[str, Any]:
if not path.exists():
return _check("ok", "not initialized")
try:
# Read-only schema query: catches unreadable/corrupt DBs without competing
# with state writers. ``closing`` is required — sqlite3's connection
# context manager only commits/rolls back, never closes, so a bare
# ``with connect()`` would leak one connection (and fds) per health poll.
uri = f"file:{path.as_posix()}?mode=ro"
with closing(sqlite3.connect(uri, uri=True, timeout=1.0)) as conn:
# Read-only schema query: catches unreadable/corrupt DBs without competing with
# writers. ``closing`` is required — sqlite3's context manager only commits/rolls
# back, never closes, so a bare ``with connect()`` leaks a connection per poll.
with closing(sqlite3.connect(f"file:{path.as_posix()}?mode=ro", uri=True, timeout=1.0)) as conn:
conn.execute("PRAGMA query_only = ON")
conn.execute("SELECT name FROM sqlite_master LIMIT 1").fetchone()
return _check("ok")
@@ -48,63 +43,48 @@ def _probe_config(home: Path) -> dict[str, Any]:
return _check("ok", "using defaults")
try:
raw = yaml.safe_load(path.read_text(encoding="utf-8"))
if raw is not None and not isinstance(raw, dict):
return _check("degraded", "top level is not a mapping")
return _check("ok")
except Exception as exc:
return _check("degraded", f"invalid config ({type(exc).__name__})")
return _check("ok") if raw is None or isinstance(raw, dict) else _check("degraded", "top level is not a mapping")
def _probe_disk(home: Path) -> dict[str, Any]:
try:
usage = shutil.disk_usage(home)
used_pct = round((usage.used / usage.total) * 100, 1) if usage.total else 0.0
status = "degraded" if used_pct >= _DISK_DEGRADED_PERCENT else "ok"
return _check(status, used_percent=used_pct, free_bytes=usage.free)
except Exception as exc:
return _check("degraded", type(exc).__name__)
used_pct = round((usage.used / usage.total) * 100, 1) if usage.total else 0.0
return _check("degraded" if used_pct >= _DISK_DEGRADED_PERCENT else "ok", used_percent=used_pct, free_bytes=usage.free)
def _probe_gateway(runtime_status: dict[str, Any]) -> dict[str, Any]:
state = str(runtime_status.get("gateway_state") or "unknown")
platforms = runtime_status.get("platforms")
if not isinstance(platforms, dict):
platforms = {}
platforms = platforms if isinstance(platforms, dict) else {}
connected = sum(
1
for value in platforms.values()
if isinstance(value, dict)
and str(value.get("state") or value.get("status") or "").lower()
in {"connected", "running", "ok"}
isinstance(v, dict) and str(v.get("state") or v.get("status") or "").lower() in _CONNECTED_STATES
for v in platforms.values()
)
status = "ok" if state in {"running", "draining"} else "degraded"
return _check(status, state=state, connected_platforms=connected, platforms=len(platforms))
return _check("ok" if state in {"running", "draining"} else "degraded", state=state,
connected_platforms=connected, platforms=len(platforms))
def _probe_session_store(runtime_status: dict[str, Any], state_db_probe: dict[str, Any]) -> dict[str, Any]:
"""Report the running gateway cache state, not an independent reopen."""
runtime_store = runtime_status.get("session_store")
if isinstance(runtime_store, dict):
state = str(runtime_store.get("status") or "unknown")
if state in {"ok", "unavailable", "retrying"}:
return _check(state)
state = str(runtime_store.get("status") or "unknown") if isinstance(runtime_store, dict) else ""
if state in {"ok", "unavailable", "retrying"}:
return _check(state)
# Older gateways publish no cache state: fall back to the state_db probe.
return _check("ok" if state_db_probe.get("status") == "ok" else "unavailable")
def collect_runtime_readiness(
*,
configured_model: str,
runtime_status: dict[str, Any] | None,
active_api_runs: int = 0,
process_completion_queue_depth: int = 0,
active_delegations: int = 0,
*, configured_model: str, runtime_status: dict[str, Any] | None, active_api_runs: int = 0,
process_completion_queue_depth: int = 0, active_delegations: int = 0,
) -> dict[str, Any]:
"""Return bounded readiness diagnostics without mutating runtime state.
Even on the authenticated endpoint, probes expose status and counts only:
never config values, credentials, paths, queue payloads, or exception messages.
"""
"""Bounded readiness diagnostics, no runtime mutation. Even authenticated, probes
expose status and counts only: never config values, credentials, paths, payloads."""
home = get_hermes_home()
runtime = runtime_status if isinstance(runtime_status, dict) else {}
state_db_probe = _probe_state_db(home)
@@ -116,14 +96,12 @@ def collect_runtime_readiness(
"disk": _probe_disk(home),
"gateway": _probe_gateway(runtime),
"background_queues": _check(
"ok",
active_api_runs=max(0, int(active_api_runs)),
"ok", active_api_runs=max(0, int(active_api_runs)),
process_completions=max(0, int(process_completion_queue_depth)),
active_delegations=max(0, int(active_delegations)),
),
}
overall = "ok" if all(item.get("status") == "ok" for item in checks.values()) else "degraded"
return {"status": overall, "checks": checks}
return {"status": "ok" if all(c.get("status") == "ok" for c in checks.values()) else "degraded", "checks": checks}
__all__ = ["collect_runtime_readiness"]
gateway/response_filters.py
+26 -56
View File
@@ -12,12 +12,7 @@ from typing import Any
# Exact whole-response markers meaning "the agent intentionally chose not to
# reply". Keep small and explicit; arbitrary empty output remains an
# error/empty-response path, not silence.
LIVE_GATEWAY_SILENT_MARKERS = frozenset({
"[SILENT]",
"SILENT",
"NO_REPLY",
"NO REPLY",
})
LIVE_GATEWAY_SILENT_MARKERS = frozenset({"[SILENT]", "SILENT", "NO_REPLY", "NO REPLY"})
# Longer than any marker could plausibly be, even with stray punctuation.
_MARKER_LENGTH_CAP = 64
@@ -42,20 +37,14 @@ def _strip_edge_silence_punctuation(text: str) -> str:
return text[start:end].strip()
def _canonical_silence_candidates(text: str) -> tuple[str, ...]:
exact = _canonical_silence_candidate(text)
stripped = _strip_edge_silence_punctuation(text.strip())
if stripped == text.strip():
return (exact,)
return (exact, _canonical_silence_candidate(stripped))
def _short_stripped(text: Any) -> str:
"""Stripped text if it is a non-empty string within the marker cap, else ''."""
if not isinstance(text, str):
return ""
stripped = text.strip()
return stripped if 0 < len(stripped) <= _MARKER_LENGTH_CAP else ""
def _canonical_silence_candidates(text: Any) -> tuple[str, ...]:
"""Canonical forms of a short marker-sized response; ``()`` when not a candidate at all."""
stripped = text.strip() if isinstance(text, str) else ""
if not 0 < len(stripped) <= _MARKER_LENGTH_CAP:
return ()
depunctuated = _strip_edge_silence_punctuation(stripped)
forms = (stripped,) if depunctuated == stripped else (stripped, depunctuated)
return tuple(_canonical_silence_candidate(f) for f in forms)
def is_intentional_silence_response(response: Any) -> bool:
@@ -64,61 +53,42 @@ def is_intentional_silence_response(response: Any) -> bool:
Prose that merely mentions ``NO_REPLY`` must be delivered normally. A blank
response is not silence either — that is the empty-response failure path.
"""
stripped = _short_stripped(response)
return bool(stripped) and any(
c in LIVE_GATEWAY_SILENT_MARKERS for c in _canonical_silence_candidates(stripped)
)
return any(c in LIVE_GATEWAY_SILENT_MARKERS for c in _canonical_silence_candidates(response))
def is_autonomous_silence_response(response: Any) -> bool:
"""Loose silence matcher for autonomous lanes (cron, webhook).
Autonomous lanes ask for ``[SILENT]`` when a tick produced nothing worth
attention, and models reliably bracket the marker with a short note. Unlike
:func:`is_intentional_silence_response` (interactive rule: EXACTLY a marker),
this suppresses when a marker is the whole response, sits on its own first or
last line, or the bracketed sentinel opens the response (``[SILENT] No
changes detected``). A token buried mid-sentence is still delivered.
Models reliably bracket ``[SILENT]`` with a short note, so unlike the
interactive EXACT rule this also suppresses when a marker sits on its own
first/last line or the bracketed sentinel opens the response (``[SILENT] No
changes detected``). A token buried mid-sentence is still delivered.
Shares :data:`LIVE_GATEWAY_SILENT_MARKERS` so the two sets cannot drift.
"""
if not isinstance(response, str):
return False
stripped = response.strip()
stripped = response.strip() if isinstance(response, str) else ""
if not stripped:
return False
def _is_token(line: str) -> bool:
return _canonical_silence_candidate(line) in LIVE_GATEWAY_SILENT_MARKERS
if _is_token(stripped):
return True
lines = [ln for ln in stripped.splitlines() if ln.strip()]
if lines and (_is_token(lines[0]) or _is_token(lines[-1])):
return True
# Bracketed form only, so a bare "Silent retry succeeded" is NOT swallowed.
return stripped.upper().startswith("[SILENT]")
# Bracketed form only for the prefix rule, so a bare "Silent retry succeeded" is NOT swallowed.
return stripped.upper().startswith("[SILENT]") or any(
_canonical_silence_candidate(c) in LIVE_GATEWAY_SILENT_MARKERS for c in (stripped, lines[0], lines[-1])
)
def is_intentional_silence_agent_result(agent_result: dict | None, response: Any) -> bool:
"""Silence markers suppress delivery only for successful agent turns."""
if not isinstance(agent_result, dict) or agent_result.get("failed"):
return False
return is_intentional_silence_response(response)
return isinstance(agent_result, dict) and not agent_result.get("failed") and is_intentional_silence_response(response)
def is_partial_silence_marker(text: Any) -> bool:
"""True while streamed ``text`` could still resolve to a silence marker.
The streaming path must decide, before the whole response is known, whether
to show its buffer. A buffer whose canonical form is a non-empty *prefix* of
a marker (``"NO"`` on the way to ``"NO_REPLY"``, or an exact marker not yet
terminated by stream-end) is held back so a raw marker is never shown and
then retracted. Anything that has diverged from every marker, or exceeds the
marker cap, returns False so normal streaming resumes. Shares the marker set
and canonicalization with :func:`is_intentional_silence_response`.
A buffer whose canonical form is a non-empty *prefix* of a marker (``"NO"`` on
the way to ``"NO_REPLY"``, or an exact marker not yet terminated by stream-end)
is held back so a raw marker is never shown and then retracted. Divergence
from every marker, or exceeding the cap, resumes normal streaming.
"""
stripped = _short_stripped(text)
return bool(stripped) and any(
return any(
c and any(marker.startswith(c) for marker in LIVE_GATEWAY_SILENT_MARKERS)
for c in _canonical_silence_candidates(stripped)
for c in _canonical_silence_candidates(text)
)
gateway/restart.py
+66 -142
View File
@@ -6,96 +6,63 @@ from collections.abc import Mapping
from hermes_cli.config import DEFAULT_CONFIG
# EX_TEMPFAIL (sysexits.h): ask the service manager to restart the gateway
# after a graceful drain/reload path completes.
# EX_TEMPFAIL (sysexits.h): ask the service manager to restart after a graceful drain/reload.
GATEWAY_SERVICE_RESTART_EXIT_CODE = 75
# EX_CONFIG (sysexits.h): fatal configuration error (token collision, no
# platforms). The s6 finish script translates this into exit 125 (permanent
# failure) so the supervisor stops restarting the gateway.
# EX_CONFIG (sysexits.h): fatal configuration error (token collision, no platforms);
# the s6 finish script maps it to exit 125 so the supervisor stops restarting.
GATEWAY_FATAL_CONFIG_EXIT_CODE = 78
# Set by ``hermes gateway run --external-supervisor``. Unlike systemd's INVOCATION_ID
# and launchd's XPC_SERVICE_NAME, this survives wrappers that replace the child
# environment (e.g. ``sudo env -i``).
EXTERNAL_GATEWAY_SUPERVISOR_ENV = "HERMES_GATEWAY_EXTERNAL_SUPERVISOR"
DEFAULT_GATEWAY_RESTART_DRAIN_TIMEOUT = float(DEFAULT_CONFIG["agent"]["restart_drain_timeout"])
DEFAULT_GATEWAY_SIGNAL_INTERRUPT_GRACE_TIMEOUT = float(DEFAULT_CONFIG["gateway"]["signal_interrupt_grace_timeout"])
DEFAULT_GATEWAY_POST_INTERRUPT_GRACE_TIMEOUT = 5.0
# In-band restart waits for active turns to finish *before* ``stop()`` begins; distinct from
# ``restart_drain_timeout``, the force-interrupt budget once ``stop()`` runs (short under TimeoutStopSec).
DEFAULT_GATEWAY_RESTART_AFTER_TURN_TIMEOUT = float(DEFAULT_CONFIG["agent"]["restart_after_turn_timeout"])
# Cron-only floor under the ``stop()`` drain. ``restart_drain_timeout`` defaults to 0 because
# interrupting a *chat* turn is cheap and recoverable (user told, session resume_pending); an
# interrupted *cron* run is a permanent failure in jobs.json — a 0s drain silently destroys work.
DEFAULT_GATEWAY_CRON_DRAIN_TIMEOUT = float(DEFAULT_CONFIG["agent"]["cron_drain_timeout"])
# Watchdog leash held back for post-drain work (interrupt agents, kill subprocesses, mark jobs,
# disconnect adapters). Waiting for cron past that trades a job killed *and recorded* for one
# SIGKILLed mid-write and wedged at ``last_status=running`` forever.
CRON_DRAIN_CLEANUP_RESERVE_S = 10.0
# systemd TimeoutStopSec headroom after the stop-path drain budget, and the floor when that
# budget is still the default immediate (0s) chat drain. Keep in lockstep with generate_systemd_unit().
SYSTEMD_STOP_HEADROOM_S = 30.0
SYSTEMD_TIMEOUT_STOP_SEC_FLOOR = 60.0
_TRUTHY = {"1", "true", "yes", "on"}
def is_global_startup_conflict(error_code: str | None) -> bool:
"""True when an adapter's fatal error is a single-writer ownership conflict.
Adapters emit ``{scope}_lock`` with ``retryable=True`` so a *mid-run*
reconnect can recover once the holder exits or a stale record is cleared.
At startup a live foreign
holder is a configuration conflict (two gateways cannot poll one token), so
the startup router must not treat it as a transient blip. Matches by error
CODE only (``lock_conflict`` / ``*_lock``), never by message text.
Adapters emit ``{scope}_lock`` with ``retryable=True`` so a *mid-run* reconnect can
recover; at startup a live foreign holder is a configuration conflict (two gateways
cannot poll one token), not a transient blip. Matches by error CODE only, never text.
"""
code = (error_code or "").strip().lower()
return bool(code) and (code == "lock_conflict" or code.endswith("_lock"))
# Set by ``hermes gateway run --external-supervisor``. Unlike systemd's
# INVOCATION_ID and launchd's XPC_SERVICE_NAME, this survives wrappers that
# replace the child environment (e.g. ``sudo env -i``).
EXTERNAL_GATEWAY_SUPERVISOR_ENV = "HERMES_GATEWAY_EXTERNAL_SUPERVISOR"
DEFAULT_GATEWAY_RESTART_DRAIN_TIMEOUT = float(
DEFAULT_CONFIG["agent"]["restart_drain_timeout"]
)
DEFAULT_GATEWAY_SIGNAL_INTERRUPT_GRACE_TIMEOUT = float(
DEFAULT_CONFIG["gateway"]["signal_interrupt_grace_timeout"]
)
DEFAULT_GATEWAY_POST_INTERRUPT_GRACE_TIMEOUT = 5.0
# In-band restart (``/restart``, SIGUSR1, self-restart) waits for active turns
# to finish *before* ``stop()`` begins. Distinct from ``restart_drain_timeout``,
# the force-interrupt budget once ``stop()`` is running (must stay short under
# systemd TimeoutStopSec).
DEFAULT_GATEWAY_RESTART_AFTER_TURN_TIMEOUT = float(
DEFAULT_CONFIG["agent"]["restart_after_turn_timeout"]
)
# Cron-only floor under the ``stop()`` drain. ``restart_drain_timeout`` defaults
# to 0 because interrupting a *chat* turn is cheap and recoverable (user is
# told, session pre-marked resume_pending). An interrupted *cron* run has
# neither property — nobody is waiting on it, it lands in jobs.json as a
# permanent failure and a recurring job just waits for its next schedule — so a
# zero-second drain silently destroys work.
DEFAULT_GATEWAY_CRON_DRAIN_TIMEOUT = float(
DEFAULT_CONFIG["agent"]["cron_drain_timeout"]
)
# Seconds of the shutdown watchdog leash held back for post-drain work
# (interrupt agents, kill tool subprocesses, mark jobs interrupted, disconnect
# adapters). Waiting for cron past that trades a job killed *and recorded* for
# one SIGKILLed mid-write and wedged at ``last_status=running`` forever.
CRON_DRAIN_CLEANUP_RESERVE_S = 10.0
# systemd TimeoutStopSec headroom after the stop-path drain budget, and the
# floor when that budget is still the default immediate (0s) chat drain.
# Keep in lockstep with generate_systemd_unit().
SYSTEMD_STOP_HEADROOM_S = 30.0
SYSTEMD_TIMEOUT_STOP_SEC_FLOOR = 60.0
def is_gateway_supervisor_process(
environ: Mapping[str, str] | None = None,
) -> bool:
def is_gateway_supervisor_process(environ: Mapping[str, str] | None = None) -> bool:
"""Return whether this gateway process is owned by a supervisor."""
env = os.environ if environ is None else environ
if env.get("INVOCATION_ID") or env.get("HERMES_S6_SUPERVISED_CHILD"):
return True
xpc_service = env.get("XPC_SERVICE_NAME", "")
if xpc_service and xpc_service != "0":
return True
return str(env.get(EXTERNAL_GATEWAY_SUPERVISOR_ENV, "")).strip().lower() in {
"1",
"true",
"yes",
"on",
}
return bool(env.get("INVOCATION_ID") or env.get("HERMES_S6_SUPERVISED_CHILD") or (xpc_service and xpc_service != "0")
or str(env.get(EXTERNAL_GATEWAY_SUPERVISOR_ENV, "")).strip().lower() in _TRUTHY)
def is_container_restart_context() -> bool:
"""Whether the gateway runs in a container (Docker/Podman): the detached
setsid restart path dies with the cgroup, so exit-75 service restart is the
only viable path. Separate function so tests can mock container detection
(a real ``/.dockerenv`` on CI otherwise flips the routing)."""
"""In a container (Docker/Podman) the detached setsid restart path dies with the cgroup,
so exit-75 service restart is the only viable path. Own function so tests can mock it."""
return os.path.exists("/.dockerenv") or os.path.exists("/run/.containerenv")
@@ -107,25 +74,21 @@ def _seconds(value: object, fallback: float = 0.0) -> float:
return fallback
def _parse_timeout_keeping_zero(raw: object, default: float) -> float:
"""Parse a timeout where ``0`` is a deliberate disable (must NOT fall
through to ``default``), unlike None / blank / non-numeric input."""
def _parse_timeout_keeping_zero(raw: object, default: float, *, finite: bool = False) -> float:
"""Parse a timeout where ``0`` is a deliberate disable (must NOT fall through
to ``default``), unlike None / blank / non-numeric (/ non-finite) input."""
if raw is None or (isinstance(raw, str) and not raw.strip()):
return default
try:
value = float(raw)
value = float(raw) # type: ignore[arg-type]
except (TypeError, ValueError):
return default
return max(0.0, value)
return default if finite and not math.isfinite(value) else max(0.0, value)
def parse_restart_drain_timeout(raw: object) -> float:
"""Parse a configured drain timeout, falling back to the shared default."""
try:
value = float(raw) if str(raw or "").strip() else DEFAULT_GATEWAY_RESTART_DRAIN_TIMEOUT
except (TypeError, ValueError):
return DEFAULT_GATEWAY_RESTART_DRAIN_TIMEOUT
return max(0.0, value)
"""Parse a configured drain timeout; falsy (incl. ``0``) falls back to the shared default."""
return _parse_timeout_keeping_zero(raw or None, DEFAULT_GATEWAY_RESTART_DRAIN_TIMEOUT)
def parse_restart_after_turn_timeout(raw: object) -> float:
@@ -138,83 +101,44 @@ def parse_cron_drain_timeout(raw: object) -> float:
return _parse_timeout_keeping_zero(raw, DEFAULT_GATEWAY_CRON_DRAIN_TIMEOUT)
def parse_signal_interrupt_grace_timeout(raw: object) -> float:
"""Parse the unexpected-signal post-interrupt grace timeout."""
return _parse_timeout_keeping_zero(raw, DEFAULT_GATEWAY_SIGNAL_INTERRUPT_GRACE_TIMEOUT, finite=True)
def resolve_cron_drain_budget(
drain_timeout: float,
cron_drain_timeout: float,
*,
watchdog_delay: float,
elapsed: float = 0.0,
drain_timeout: float, cron_drain_timeout: float, *, watchdog_delay: float, elapsed: float = 0.0,
cleanup_reserve_s: float = CRON_DRAIN_CLEANUP_RESERVE_S,
) -> float:
"""Seconds the shutdown drain may spend waiting on in-flight cron work.
"""Seconds the stop drain may wait on in-flight cron work.
The configured floor is clamped to what this process can honour: the
shutdown watchdog hard-exits at ``watchdog_delay`` (and TimeoutStopSec is
sized from the same budget), so waiting past that leash minus
``cleanup_reserve_s`` swaps a cleanly-interrupted job for a SIGKILL that
leaves it wedged. Never returns less than ``drain_timeout``: the cron floor
only ever extends the wait, so an operator who deliberately configured a
long ``restart_drain_timeout`` keeps it.
Clamped to what this process can honour: the watchdog hard-exits at ``watchdog_delay``,
so waiting past that leash minus ``cleanup_reserve_s`` swaps a cleanly-interrupted job
for a SIGKILL that leaves it wedged. Never less than ``drain_timeout`` (only extends).
"""
drain = _seconds(drain_timeout)
floor = _seconds(cron_drain_timeout)
if floor <= 0.0:
return drain
ceiling = (
_seconds(watchdog_delay)
- _seconds(elapsed)
- _seconds(cleanup_reserve_s, CRON_DRAIN_CLEANUP_RESERVE_S)
)
ceiling = _seconds(watchdog_delay) - _seconds(elapsed) - _seconds(cleanup_reserve_s, CRON_DRAIN_CLEANUP_RESERVE_S)
return max(drain, min(floor, ceiling))
def resolve_systemd_timeout_stop_sec(
drain_timeout: float,
cron_drain_timeout: float = DEFAULT_GATEWAY_CRON_DRAIN_TIMEOUT,
*,
cleanup_reserve_s: float = CRON_DRAIN_CLEANUP_RESERVE_S,
headroom_s: float = SYSTEMD_STOP_HEADROOM_S,
drain_timeout: float, cron_drain_timeout: float = DEFAULT_GATEWAY_CRON_DRAIN_TIMEOUT, *,
cleanup_reserve_s: float = CRON_DRAIN_CLEANUP_RESERVE_S, headroom_s: float = SYSTEMD_STOP_HEADROOM_S,
floor_s: float = SYSTEMD_TIMEOUT_STOP_SEC_FLOOR,
) -> int:
"""Seconds systemd ``TimeoutStopSec`` must cover the full stop budget.
``restart_drain_timeout`` is only the chat-turn interrupt budget (default 0);
the stop path may first wait ``cron_drain_timeout`` + ``cleanup_reserve_s``
for cron work, so sizing from drain alone lets systemd SIGKILL an in-budget
drain. A zero cron timeout is an opt-out and does not extend the budget.
Non-numeric inputs degrade to 0.
"""
"""Seconds systemd ``TimeoutStopSec`` must cover: the stop path may first wait
``cron_drain_timeout`` + ``cleanup_reserve_s`` for cron work, so sizing from the chat drain
alone lets systemd SIGKILL an in-budget drain. A zero cron timeout is an opt-out."""
drain = _seconds(drain_timeout)
cron = _seconds(cron_drain_timeout)
cron_budget = (cron + _seconds(cleanup_reserve_s)) if cron > 0.0 else 0.0
stop_budget = max(drain, cron_budget)
return int(max(_seconds(floor_s), stop_budget + _seconds(headroom_s)))
return int(max(_seconds(floor_s), max(drain, cron_budget) + _seconds(headroom_s)))
def resolve_restart_exit_wait_budget(
drain_timeout: float,
after_turn_timeout: float,
*,
headroom: float = 15.0,
) -> float:
"""Seconds a CLI should wait for the gateway PID to exit after SIGUSR1.
In-band restart may defer ``stop()`` until turns finish (``after_turn_timeout``)
and then spend ``drain_timeout`` inside ``stop()``; callers that hard-kill on
expiry must cover both phases.
"""
def resolve_restart_exit_wait_budget(drain_timeout: float, after_turn_timeout: float, *, headroom: float = 15.0) -> float:
"""Seconds a CLI should wait for the gateway PID to exit after SIGUSR1: in-band restart may
defer ``stop()`` until turns finish, then spend ``drain_timeout`` inside it — cover both."""
return _seconds(drain_timeout) + _seconds(after_turn_timeout) + _seconds(headroom)
def parse_signal_interrupt_grace_timeout(raw: object) -> float:
"""Parse the unexpected-signal post-interrupt grace timeout."""
try:
if raw is None or (isinstance(raw, str) and not raw.strip()):
value = DEFAULT_GATEWAY_SIGNAL_INTERRUPT_GRACE_TIMEOUT
else:
value = float(raw)
except (TypeError, ValueError):
return DEFAULT_GATEWAY_SIGNAL_INTERRUPT_GRACE_TIMEOUT
if not math.isfinite(value):
return DEFAULT_GATEWAY_SIGNAL_INTERRUPT_GRACE_TIMEOUT
return max(0.0, value)
gateway/restart_loop_guard.py
+36 -78
View File
@@ -1,35 +1,23 @@
"""Auto-resume restart-loop breaker (defense-3).
Defenses 1 and 2 (the ``_HERMES_GATEWAY`` guard on ``hermes gateway
stop|restart`` + ``terminal_tool``, and the cron-creation lifecycle filter)
stop the agent scheduling its own restart via cron/CLI. They do NOT cover
every SIGTERM source (raw ``launchctl kickstart``, a bad external monitor,
any repeated crash): the supervisor respawns, the gateway auto-resumes the
restart-interrupted session, whose next turn re-runs the offending logic.
This module is the last-resort circuit breaker. Each boot with
restart-interrupted sessions pending is timestamped and persisted (each boot
is a fresh process, so in-memory state is useless). Boots CHAIN while
consecutive gaps stay within ``max_gap_seconds``, so slow crash cycles (a
wedged loop killed by the liveness watchdog every ~150s) trip exactly like the
fast ~10s respawn loop. When tripped, the caller SKIPS auto-resume for that
boot — the gateway still serves real inbound messages, it just stops replaying
the session that keeps killing it.
State lives in ``<HERMES_HOME>/gateway/restart_loop.json`` (profile-scoped).
Best-effort: any read/write failure fails OPEN (no false trip) because a
broken breaker must never wedge a healthy gateway.
Defenses 1-2 (``_HERMES_GATEWAY`` guard on ``hermes gateway stop|restart`` /
``terminal_tool``, cron lifecycle filter) stop the agent scheduling its own restart
but not every SIGTERM source: the supervisor respawns, the gateway auto-resumes the
restart-interrupted session, whose next turn re-runs the offending logic. Boots are
persisted to ``<HERMES_HOME>/gateway/restart_loop.json`` and CHAIN while gaps stay
within ``max_gap_seconds`` (a ~150s watchdog-kill cycle trips like a ~10s loop).
Tripped → caller SKIPS auto-resume. Any I/O failure fails OPEN, never wedging.
"""
from __future__ import annotations
import contextlib
import json
import logging
import time
from typing import List, Optional
from hermes_constants import get_hermes_home
import contextlib
logger = logging.getLogger("gateway.run")
@@ -37,16 +25,12 @@ logger = logging.getLogger("gateway.run")
# within a few cycles.
DEFAULT_MAX_RESTARTS = 3
DEFAULT_WINDOW_SECONDS = 60
# Longest gap between consecutive restart-interrupted boots that still counts
# them as the SAME loop. A fixed-window prune only sees cycles faster than the
# window (a slower loop drops its own history every boot and never trips);
# chaining on the inter-boot gap makes the breaker period-agnostic, and a
# single boot followed by real quiet resets the chain.
# Longest gap between consecutive restart-interrupted boots still counted as the
# SAME loop. A fixed-window prune only sees cycles faster than the window (a slower
# loop drops its history every boot and never trips); chaining on the inter-boot
# gap is period-agnostic, and real quiet resets it.
DEFAULT_MAX_GAP_SECONDS = 300
# Cap the persisted chain; only the newest ``max_restarts`` entries can change
# a verdict, the rest are forensics.
# Only the newest ``max_restarts`` entries can change a verdict; the rest are forensics.
_MAX_STORED_BOOTS = 50
@@ -63,57 +47,44 @@ def _load_boots() -> List[float]:
def _save_boots(boots: List[float]) -> None:
try:
with contextlib.suppress(OSError):
path = _state_path()
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps({"boots": boots}), encoding="utf-8")
except OSError:
pass
def _chain_gap(window_seconds: int, max_gap_seconds: int) -> float:
"""Inter-boot gap that still links two boots. Floored by ``window_seconds`` so
"""Inter-boot gap that still links two boots; floored by ``window_seconds`` so
widening the window never makes the breaker *less* sensitive."""
return float(max(1, window_seconds, max_gap_seconds))
def _chain_ending_at(boots: List[float], ts: float, gap: float) -> List[float]:
"""Unbroken chain of boots leading up to ``ts`` (oldest first).
Walks backwards keeping boots while each successive gap stays within
``gap``; the first wider gap ends the chain (older boots belong to an
already-resolved episode). Nothing recent enough -> empty list, which is how
a healthy gateway forgets an old loop.
"""
"""Unbroken chain of boots leading up to ``ts`` (oldest first): walks backwards
while each gap stays within ``gap``; the first wider gap ends the chain (older
boots are a resolved episode). Empty when nothing is recent — how a healthy
gateway forgets a loop."""
chain: List[float] = []
prev = ts
for t in sorted(boots, reverse=True):
if t > ts:
# Clock moved backwards (NTP step, restored state file): treat the
# future entry as adjacent rather than dropping the whole chain.
if t > ts: # clock moved backwards (NTP step, restored file): future entry is adjacent, not a break
chain.append(t)
continue
if prev - t > gap:
break
chain.append(t)
prev = t
chain.reverse()
return chain
return chain[::-1]
def record_restart_interrupted_boot(
window_seconds: int = DEFAULT_WINDOW_SECONDS,
*,
now: Optional[float] = None,
window_seconds: int = DEFAULT_WINDOW_SECONDS, *, now: Optional[float] = None,
max_gap_seconds: int = DEFAULT_MAX_GAP_SECONDS,
) -> List[float]:
"""Record a restart-interrupted boot; return the pruned chain + now (most recent last).
Best-effort — a persistence failure returns the in-memory list without raising.
"""
"""Record a restart-interrupted boot; return the pruned chain + now (most recent
last). A persistence failure returns the in-memory list without raising."""
ts = time.time() if now is None else now
boots = _chain_ending_at(_load_boots(), ts, _chain_gap(window_seconds, max_gap_seconds))
boots.append(ts)
boots = _chain_ending_at(_load_boots(), ts, _chain_gap(window_seconds, max_gap_seconds)) + [ts]
_save_boots(boots[-_MAX_STORED_BOOTS:])
return boots
@@ -125,32 +96,19 @@ def clear() -> None:
def check_and_record(
max_restarts: int = DEFAULT_MAX_RESTARTS,
window_seconds: int = DEFAULT_WINDOW_SECONDS,
*,
now: Optional[float] = None,
max_gap_seconds: int = DEFAULT_MAX_GAP_SECONDS,
max_restarts: int = DEFAULT_MAX_RESTARTS, window_seconds: int = DEFAULT_WINDOW_SECONDS, *,
now: Optional[float] = None, max_gap_seconds: int = DEFAULT_MAX_GAP_SECONDS,
) -> bool:
"""Record this boot and return True when auto-resume should be SKIPPED.
The single entry point the gateway calls: appends the current boot, then
checks whether the updated chain has reached ``max_restarts``.
"""
boots = record_restart_interrupted_boot(
window_seconds, now=now, max_gap_seconds=max_gap_seconds
)
tripped = len(boots) >= max_restarts if max_restarts > 0 else False
"""Gateway entry point: record this boot; True when the chain reached
``max_restarts`` and auto-resume should be SKIPPED."""
boots = record_restart_interrupted_boot(window_seconds, now=now, max_gap_seconds=max_gap_seconds)
tripped = max_restarts > 0 and len(boots) >= max_restarts
if tripped:
logger.warning(
"Restart-loop breaker TRIPPED: %d chained restart-interrupted "
"gateway boots (no gap wider than %ds; threshold %d). Skipping "
"auto-resume to break a suspected SIGTERM-respawn loop (#30719, "
"#81642). Restart-interrupted sessions stay resume-pending and "
"will continue on the next real user message. If this is a false "
"positive, delete %s.",
len(boots),
int(_chain_gap(window_seconds, max_gap_seconds)),
max_restarts,
_state_path(),
"Restart-loop breaker TRIPPED: %d chained restart-interrupted gateway boots (no gap wider than %ds; "
"threshold %d). Skipping auto-resume to break a suspected SIGTERM-respawn loop (#30719, #81642). "
"Restart-interrupted sessions stay resume-pending and will continue on the next real user message. "
"If this is a false positive, delete %s.",
len(boots), int(_chain_gap(window_seconds, max_gap_seconds)), max_restarts, _state_path(),
)
return tripped
gateway/rich_sent_store.py
+9 -21
View File
@@ -1,13 +1,8 @@
"""Local index of text we've sent via ``sendRichMessage`` (Bot API 10.1).
Telegram does NOT echo a rich message's content back in ``reply_to_message``
(``.text``/``.caption`` empty, ``.api_kwargs`` None), so replies to rich sends
arrive with no quotable text. We remember ``message_id -> text`` at send time
and look it up by ``reply_to_id`` on inbound. This module is the single source
of truth for that index.
Best-effort and dependency-free: every operation swallows errors and degrades
to a no-op / ``None`` so it can never break a send or an inbound message.
"""Local index of text we've sent via ``sendRichMessage`` (Bot API 10.1). Telegram does NOT echo a rich
message's content back in ``reply_to_message`` (``.text``/``.caption`` empty, ``.api_kwargs`` None), so
replies to rich sends arrive with no quotable text; we remember ``message_id -> text`` at send time and
look it up by ``reply_to_id`` on inbound. Best-effort and dependency-free: every operation swallows
errors and degrades to a no-op / ``None`` so it can never break a send or an inbound message.
"""
from __future__ import annotations
@@ -22,16 +17,10 @@ _MAX_TEXT_CHARS = 2000
def _store_path() -> str:
# get_hermes_home() honors the active profile override.
from hermes_constants import get_hermes_home
from hermes_constants import get_hermes_home # honors the active profile override
return os.path.join(str(get_hermes_home()), "state", "rich_sent_index.json")
def _key(chat_id, message_id) -> str:
return f"{chat_id}:{message_id}"
def _load(path: str) -> dict:
try:
with open(path, "r", encoding="utf-8") as fh:
@@ -49,10 +38,9 @@ def record(chat_id, message_id, text: Optional[str]) -> None:
try:
os.makedirs(os.path.dirname(path), exist_ok=True)
data = _load(path)
data[_key(chat_id, message_id)] = {"t": text[:_MAX_TEXT_CHARS], "ts": int(time.time())}
data[f"{chat_id}:{message_id}"] = {"t": text[:_MAX_TEXT_CHARS], "ts": int(time.time())}
if len(data) > _MAX_ENTRIES: # trim oldest by timestamp
by_age = sorted(data.items(), key=lambda kv: kv[1].get("ts", 0))
for k, _ in by_age[: len(data) - _MAX_ENTRIES]:
for k, _ in sorted(data.items(), key=lambda kv: kv[1].get("ts", 0))[: len(data) - _MAX_ENTRIES]:
data.pop(k, None)
tmp = f"{path}.tmp.{os.getpid()}"
with open(tmp, "w", encoding="utf-8") as fh:
@@ -66,5 +54,5 @@ def lookup(chat_id, message_id) -> Optional[str]:
"""Return stored text for ``(chat_id, message_id)`` or ``None``."""
if message_id is None or chat_id is None:
return None
entry = _load(_store_path()).get(_key(chat_id, message_id))
entry = _load(_store_path()).get(f"{chat_id}:{message_id}")
return (entry.get("t") or None) if isinstance(entry, dict) else None
gateway/run.py
+1555 -3495
View File
File diff suppressed because it is too large Load Diff
tests/gateway/test_api_server_runs.py
+1 -2
View File
@@ -1874,8 +1874,7 @@ class TestHostedRoomRuns:
for target in (
"gateway.platforms.api_server.time.time",
"gateway.hosted_room_peer.time.time",
"gateway.hosted_rooms.time.time",
"gateway.hosted_rooms_common.time.time",
):
monkeypatch.setattr(target, lambda: 200)
claims = {