diff --git a/gateway/run_adapters.py b/gateway/run_adapters.py index 1c15a1d9aa..608f260da8 100644 --- a/gateway/run_adapters.py +++ b/gateway/run_adapters.py @@ -1,8 +1,8 @@ -"""Adapter connect/disconnect, fatal-error recovery, reconnect watcher and multiplex profile adapter methods for GatewayRunner. +"""Adapter connect/disconnect, fatal-error recovery, reconnect watcher and multiplex profile +adapters for GatewayRunner (mixin bound via the MRO). -Split out of ``gateway/run.py``; bound onto ``GatewayRunner`` via the MRO. -``gateway.run`` internals are imported lazily inside method bodies (import cycle), -so ``patch("gateway.run.X")`` keeps intercepting them at call time. +``gateway.run`` internals are imported lazily inside method bodies (import cycle), so +``patch("gateway.run.X")`` keeps intercepting them at call time. """ from __future__ import annotations @@ -10,6 +10,8 @@ from __future__ import annotations import logging from typing import TYPE_CHECKING import asyncio +import contextlib +import functools import os import time import weakref as _weakref @@ -31,22 +33,15 @@ logger = logging.getLogger("gateway.run") class GatewayAdapterLifecycleMixin: - """Adapter connect/disconnect, fatal-error recovery, reconnect watcher and multiplex profile adapter methods for GatewayRunner.""" + """Adapter lifecycle: connect/teardown, fatal recovery, reconnect watcher, multiplex profiles.""" - async def _await_adapter_cleanup_with_timeout( - self, awaitable: Awaitable[Any], timeout: float - ) -> bool: - """Wait for adapter cleanup without letting cancellation swallowing hang us. + @staticmethod + async def _wait_or_detach(task: "asyncio.Future", timeout: float) -> bool: + """Wait up to ``timeout`` for ``task``; on deadline (or our own cancellation) detach it. - ``asyncio.wait_for`` cancels an overdue child but then waits for it to exit. An adapter - close path that catches ``CancelledError`` can therefore block recovery forever. Keep - ownership of the old task through its done callback, but release the runner at the deadline. + Not ``asyncio.wait_for``: that WAITS for the cancelled child to exit, so a connect()/close() + that swallows ``CancelledError`` blocks recovery forever. Returns True if it finished in time. """ - if timeout <= 0: - await awaitable - return True - - task = asyncio.ensure_future(awaitable) try: done, _pending = await asyncio.wait({task}, timeout=timeout) except asyncio.CancelledError: @@ -54,56 +49,52 @@ class GatewayAdapterLifecycleMixin: task.add_done_callback(consume_detached_task_result) raise if task in done: - await task return True - task.cancel() task.add_done_callback(consume_detached_task_result) return False - async def _safe_adapter_disconnect(self, adapter, platform) -> None: - """Call adapter.disconnect() defensively, swallowing any error. + async def _await_adapter_cleanup_with_timeout(self, awaitable: Awaitable[Any], timeout: float) -> bool: + """Await adapter cleanup with a detach-on-deadline bound; True when it completed.""" + if timeout <= 0: + await awaitable + return True + task = asyncio.ensure_future(awaitable) + if not await self._wait_or_detach(task, timeout): + return False + await task + return True - For a failed/raised connect(): partial resources (aiohttp.ClientSession, poll tasks, child - subprocesses) would otherwise leak. Must tolerate partial-init state and never raise. + async def _safe_adapter_disconnect(self, adapter, platform) -> None: + """Call adapter.disconnect() defensively (bounded, never raises). + + After a failed connect() partial resources (ClientSession, poll tasks, subprocesses) would + otherwise leak; must tolerate partial-init state. """ timeout = self._adapter_disconnect_timeout_secs() + label = platform.value if platform is not None else "adapter" try: - completed = await self._await_adapter_cleanup_with_timeout( - adapter.disconnect(), timeout - ) - if not completed: + if not await self._await_adapter_cleanup_with_timeout(adapter.disconnect(), timeout): logger.warning( "Timed out after %.1fs while disconnecting %s adapter; continuing shutdown", - timeout, - platform.value if platform is not None else "adapter", + timeout, label, ) except Exception as e: - logger.debug( - "Defensive %s disconnect after failed connect raised: %s", - platform.value if platform is not None else "adapter", - e, - ) + logger.debug("Defensive %s disconnect after failed connect raised: %s", label, e) - async def _bounded_adapter_teardown( - self, adapter, platform, *, profile: Optional[str] = None - ) -> None: - """Tear down one adapter on the shutdown path with bounded awaits. + async def _bounded_adapter_teardown(self, adapter, platform, *, profile: Optional[str] = None) -> None: + """Tear down one adapter on the shutdown path with bounded awaits (never raises). - ``cancel_background_tasks()`` and ``disconnect()`` can block forever on half-dead network - state (e.g. a wedged WebSocket thread), stalling shutdown past systemd's ``TimeoutStopSec``; - the SIGKILL skips ``atexit`` PID-file cleanup and the next start dies with "PID file race - lost". Each await uses ``HERMES_GATEWAY_ADAPTER_DISCONNECT_TIMEOUT``; on timeout the task is - cancelled and detached so a cancellation-swallowing adapter can't hang the loop. Never raises. + Unbounded, a half-dead transport stalls shutdown past systemd's ``TimeoutStopSec``; the + SIGKILL skips ``atexit`` PID-file cleanup and the next start dies with "PID file race lost". """ timeout = self._adapter_disconnect_timeout_secs() suffix = f" (profile: {profile})" if profile else "" started_at = time.monotonic() try: - cancelled = await self._await_adapter_cleanup_with_timeout( + if not await self._await_adapter_cleanup_with_timeout( adapter.cancel_background_tasks(), timeout - ) - if not cancelled: + ): logger.warning( "✗ %s background-task cancel timed out after %.1fs - forcing continue%s", platform.value, timeout, suffix, @@ -111,13 +102,9 @@ class GatewayAdapterLifecycleMixin: except Exception as e: logger.debug("✗ %s background-task cancel error%s: %s", platform.value, suffix, e) try: - disconnected = await self._await_adapter_cleanup_with_timeout( - adapter.disconnect(), timeout - ) - if disconnected: + if await self._await_adapter_cleanup_with_timeout(adapter.disconnect(), timeout): logger.info( - "✓ %s disconnected (%.2fs)%s", - platform.value, time.monotonic() - started_at, suffix, + "✓ %s disconnected (%.2fs)%s", platform.value, time.monotonic() - started_at, suffix, ) else: logger.warning( @@ -130,109 +117,62 @@ class GatewayAdapterLifecycleMixin: platform.value, time.monotonic() - started_at, suffix, e, ) + @staticmethod + def _env_timeout_override(name: str) -> Optional[float]: + """Non-negative float from env var ``name``; None when unset or unparseable (warned).""" + raw = os.getenv(name, "").strip() + if not raw: + return None + try: + return max(0.0, float(raw)) + except ValueError: + logger.warning("Ignoring invalid %s=%r", name, raw) + return None + def _adapter_disconnect_timeout_secs(self) -> float: """Return the per-adapter disconnect timeout used during shutdown.""" from gateway.run import _ADAPTER_DISCONNECT_TIMEOUT_SECS_DEFAULT - raw = os.getenv("HERMES_GATEWAY_ADAPTER_DISCONNECT_TIMEOUT", "").strip() - if raw: - try: - timeout = float(raw) - except ValueError: - logger.warning( - "Ignoring invalid HERMES_GATEWAY_ADAPTER_DISCONNECT_TIMEOUT=%r", - raw, - ) - else: - return max(0.0, timeout) - return _ADAPTER_DISCONNECT_TIMEOUT_SECS_DEFAULT + override = self._env_timeout_override("HERMES_GATEWAY_ADAPTER_DISCONNECT_TIMEOUT") + return _ADAPTER_DISCONNECT_TIMEOUT_SECS_DEFAULT if override is None else override def _platform_connect_timeout_secs(self, platform=None, *, initial: bool = False) -> float: - """Return the per-platform connect timeout used during startup/retry. - - Telegram's full 180s connect budget is deliberately NOT spent at cold start: an unreachable - Telegram would hold the gateway out of ``running`` for the whole budget. The cold-start wait - is capped and the platform handed to the reconnect watcher, which retries with the full - budget and ``is_reconnect=True`` (preserving the offline update queue). - """ + """Per-platform connect timeout. Telegram's full 180s is NOT spent at cold start (it would + hold the gateway out of ``running``); the watcher retries with the full budget.""" from gateway.run import ( - _PLATFORM_CONNECT_TIMEOUT_SECS_DEFAULT, - _TELEGRAM_CONNECT_TIMEOUT_SECS_DEFAULT, + _PLATFORM_CONNECT_TIMEOUT_SECS_DEFAULT, _TELEGRAM_CONNECT_TIMEOUT_SECS_DEFAULT, _TELEGRAM_INITIAL_CONNECT_TIMEOUT_SECS_DEFAULT, ) - raw = os.getenv("HERMES_GATEWAY_PLATFORM_CONNECT_TIMEOUT", "").strip() - if raw: - try: - timeout = float(raw) - except ValueError: - logger.warning( - "Ignoring invalid HERMES_GATEWAY_PLATFORM_CONNECT_TIMEOUT=%r", - raw, - ) - else: - return max(0.0, timeout) - if platform == Platform.TELEGRAM: - if initial: - return _TELEGRAM_INITIAL_CONNECT_TIMEOUT_SECS_DEFAULT - return _TELEGRAM_CONNECT_TIMEOUT_SECS_DEFAULT - return _PLATFORM_CONNECT_TIMEOUT_SECS_DEFAULT + override = self._env_timeout_override("HERMES_GATEWAY_PLATFORM_CONNECT_TIMEOUT") + if override is not None: + return override + if platform != Platform.TELEGRAM: + return _PLATFORM_CONNECT_TIMEOUT_SECS_DEFAULT + return _TELEGRAM_INITIAL_CONNECT_TIMEOUT_SECS_DEFAULT if initial else _TELEGRAM_CONNECT_TIMEOUT_SECS_DEFAULT async def _connect_adapter_with_timeout( self, adapter, platform, *, is_reconnect: bool = False, initial: bool = False ) -> bool: - """Connect an adapter without allowing one platform to block others. - - ``is_reconnect`` lets adapters distinguish a cold first boot (drop any stale server-side - queue) from a watcher reconnect (preserve the queue so interim messages aren't dropped). - ``initial`` selects the capped cold-start budget for platforms whose full connect budget is - too long to spend before the gateway reaches ``running`` (Telegram's 180s). - """ + """Connect with a bound so one platform can't block others. ``is_reconnect``: cold boot + drops the stale server-side queue, a reconnect keeps it. ``initial``: capped budget.""" timeout = self._platform_connect_timeout_secs(platform, initial=initial) if timeout <= 0: return await adapter.connect(is_reconnect=is_reconnect) - # Detach-on-timeout rather than plain asyncio.wait_for: wait_for cancels the overdue task but - # then waits for it to exit, so a connect() that catches CancelledError blocks recovery - # forever (watcher never retries). Keep ownership via its done callback; release at deadline. - task = asyncio.ensure_future( - adapter.connect(is_reconnect=is_reconnect) - ) - try: - done, _pending = await asyncio.wait({task}, timeout=timeout) - except asyncio.CancelledError: - task.cancel() - task.add_done_callback(consume_detached_task_result) - raise - if task in done: - result = await task - return bool(result) - task.cancel() - task.add_done_callback(consume_detached_task_result) - raise TimeoutError( - f"{platform.value} connect timed out after {timeout:g}s" - ) + task = asyncio.ensure_future(adapter.connect(is_reconnect=is_reconnect)) + if await self._wait_or_detach(task, timeout): + return bool(await task) + raise TimeoutError(f"{platform.value} connect timed out after {timeout:g}s") async def _connect_initial_adapter_with_timeout(self, adapter, platform) -> bool: - """Connect one cold-start adapter with tightly scoped replace intent. - - The capability is visible only while this initial connect is awaited. Reconnects call - ``_connect_adapter_with_timeout`` directly and adapters also default to deny, so a later - network recovery can never evict a healthy token holder. - """ - adapter._platform_lock_takeover_allowed = bool( - self._platform_lock_takeover_on_start - ) + """Cold-start connect with replace intent visible ONLY during this await, so a later + network recovery can never evict a healthy token holder.""" + adapter._platform_lock_takeover_allowed = bool(self._platform_lock_takeover_on_start) try: - return await self._connect_adapter_with_timeout( - adapter, platform, initial=True - ) + return await self._connect_adapter_with_timeout(adapter, platform, initial=True) finally: adapter._platform_lock_takeover_allowed = False async def _handle_reaction_event(self, ctx: Dict[str, Any]) -> None: - """Fan a normalised platform reaction event out to the HookRegistry. - - The adapter-supplied ``event_name`` ("reaction:added"/"reaction:removed") is the hook event, - matching the ``agent:*`` naming scheme. Errors never block the adapter's event loop. - """ + """Fan a normalised reaction event out to the HookRegistry; errors never block the adapter.""" event_name = str(ctx.get("event_name") or "reaction:added") try: await self.hooks.emit(event_name, ctx) @@ -240,13 +180,10 @@ class GatewayAdapterLifecycleMixin: logger.debug("[Gateway] reaction hook emit failed", exc_info=True) async def _handle_adapter_fatal_error(self, adapter: BasePlatformAdapter) -> None: - """React to an adapter failure after startup. + """React to an adapter failure after startup (retryable → background reconnect queue). - Retryable errors (network blip, DNS) queue the platform for background reconnection. - The notification arrives on the failing adapter's own polling task, and the disconnect in - the handler can cancel that task mid-flight (disconnect()'s current-task guard misses it - because _safe_adapter_disconnect closes in a wrapper task), stranding the platform between - the fatal log and the reconnect queue — so the real work runs in a detached task. + Detached task: the notification arrives on the failing adapter's own polling task, which + the handler's disconnect can cancel mid-flight, stranding the platform half-handled. """ tasks = getattr(self, "_fatal_handler_tasks", None) if tasks is None: @@ -254,17 +191,14 @@ class GatewayAdapterLifecycleMixin: task = asyncio.create_task(self._handle_adapter_fatal_error_detached(adapter)) tasks.add(task) task.add_done_callback(tasks.discard) - # Await so callers that expect completion still get it — but through shield(): Task.cancel() - # on the caller also cancels the future it is awaiting (_fut_waiter), so a plain `await - # task` would tunnel the cancellation straight into the "detached" task. shield() absorbs - # it: the caller sees CancelledError, the handler runs to completion. + # shield(): a plain `await task` would tunnel the caller's cancellation into the detached + # task; with shield the caller sees CancelledError and the handler runs to completion. await asyncio.shield(task) def _queue_retryable_fatal_platform(self, adapter: BasePlatformAdapter) -> bool: - """Queue a retryable fatal adapter for background reconnection. + """Queue a retryable fatal adapter for background reconnection (True when newly queued). - Returns True when newly queued; idempotent if already queued. Must not await: callers - invoke this *before* any disconnect await so a wedged close cannot strand the platform. + Must not await: callers run this BEFORE any disconnect so a wedged close can't strand it. """ if not adapter.fatal_error_retryable: return False @@ -272,11 +206,9 @@ class GatewayAdapterLifecycleMixin: if not platform_config: return False if adapter.platform in self._failed_platforms: - # Nothing to enqueue — but "already queued" is exactly when the watcher may have died, - # and the enqueue branch below holds the ONLY _ensure_reconnect_watcher_running() call. - # _spawn_supervised gives up after _MAX_SUPERVISED_RESTARTS; without this backstop a - # queued platform is a silent permanent outage (nothing retries, and the stranded check - # treats a queued platform as safe so the process never restarts either). + # Already queued is exactly when the watcher may have died (_spawn_supervised gives up + # after _MAX_SUPERVISED_RESTARTS); without this backstop the platform is a silent + # permanent outage — nothing retries and the stranded check treats "queued" as safe. self._ensure_reconnect_watcher_running() return False self._failed_platforms[adapter.platform] = { @@ -284,148 +216,103 @@ class GatewayAdapterLifecycleMixin: "attempts": 0, "next_retry": time.monotonic(), "queued_at": time.monotonic(), - "credential_claim": self._adapter_credential_claim( - adapter.platform, adapter - ), - "listener_claim": self._adapter_listener_claim( - adapter.platform, adapter - ), + "credential_claim": self._adapter_credential_claim(adapter.platform, adapter), + "listener_claim": self._adapter_listener_claim(adapter.platform, adapter), } - logger.info( - "%s queued for background reconnection", - adapter.platform.value, - ) - # Ensure the reconnect watcher is alive — respawn if it died (e.g. restart budget exhausted) - # so queued platforms are not permanently stranded. + logger.info("%s queued for background reconnection", adapter.platform.value) self._ensure_reconnect_watcher_running() return True - async def _handle_adapter_fatal_error_detached( - self, adapter: BasePlatformAdapter - ) -> None: - """Run the fatal handler; if the platform still ends up stranded (not reconnected, not - queued, not intentionally disabled), exit the gateway with failure so the service manager - restarts it instead of leaving a silent partial outage.""" + async def _handle_adapter_fatal_error_detached(self, adapter: BasePlatformAdapter) -> None: + """Run the fatal handler; a platform left stranded (not reconnected, not queued, not + intentionally disabled) exits the gateway with failure so the service manager restarts it.""" try: - # Outer hard deadline: even with queue-before-disconnect, a hang anywhere in the impl - # (status write side effects, detach races, etc.) must not leave this task wedged - # forever — the stranded check in ``finally`` only runs when we return. + # Outer hard deadline: a hang anywhere in the impl must not wedge this task — the + # stranded check in ``finally`` only runs when we return. timeout = self._adapter_disconnect_timeout_secs() if timeout <= 0: await self._handle_adapter_fatal_error_impl(adapter) else: - # Disconnect budget plus a little queue/status bookkeeping overhead; keep the extra - # proportional so tests that shrink the disconnect timeout still finish promptly. + # Disconnect budget + proportional bookkeeping overhead (tests shrink the timeout). outer = timeout + min(2.0, max(0.05, timeout)) - completed = await self._await_adapter_cleanup_with_timeout( - self._handle_adapter_fatal_error_impl(adapter), - outer, - ) - if not completed: + if not await self._await_adapter_cleanup_with_timeout( + self._handle_adapter_fatal_error_impl(adapter), outer + ): logger.error( "Fatal-error handling for %s timed out after %.1fs; " - "ensuring reconnect queue is populated", - adapter.platform.value, - outer, + "ensuring reconnect queue is populated", adapter.platform.value, outer, ) self._queue_retryable_fatal_platform(adapter) except asyncio.CancelledError: - # Best-effort queue before re-raising: a cancelled fatal handler - # must not strand a retryable platform (#80598). - try: - self._queue_retryable_fatal_platform(adapter) - except Exception: - logger.debug( - "Failed to queue %s after fatal-handler cancellation", - adapter.platform.value, - exc_info=True, - ) + # A cancelled or raising fatal handler must not strand a retryable platform. + self._queue_retryable_best_effort(adapter, "cancellation") raise except Exception: - logger.exception( - "Fatal-error handling for %s raised unexpectedly", - adapter.platform.value, - ) - # Best-effort queue so an unexpected raise mid-handler cannot - # leave a retryable platform permanently deaf (#80598). - try: - self._queue_retryable_fatal_platform(adapter) - except Exception: - logger.debug( - "Failed to queue %s after fatal-handler exception", - adapter.platform.value, - exc_info=True, - ) + logger.exception("Fatal-error handling for %s raised unexpectedly", adapter.platform.value) + self._queue_retryable_best_effort(adapter, "exception") finally: platform = adapter.platform shutdown_event = getattr(self, "_shutdown_event", None) - stranded = ( + if ( adapter.fatal_error_retryable and platform not in self.adapters and platform not in getattr(self, "_failed_platforms", {}) and not (shutdown_event is not None and shutdown_event.is_set()) - ) - if stranded: + ): logger.error( "%s adapter was lost without entering the reconnection " - "queue; exiting gateway so the service manager restarts it.", - platform.value, - ) - self._exit_reason = ( - f"{platform.value} adapter lost without reconnection queue" + "queue; exiting gateway so the service manager restarts it.", platform.value, ) + self._exit_reason = f"{platform.value} adapter lost without reconnection queue" self._exit_with_failure = True await self.stop() + def _queue_retryable_best_effort(self, adapter: BasePlatformAdapter, why: str) -> None: + try: + self._queue_retryable_fatal_platform(adapter) + except Exception: + logger.debug( + "Failed to queue %s after fatal-handler %s", adapter.platform.value, why, exc_info=True, + ) + async def _handle_adapter_fatal_error_impl(self, adapter: BasePlatformAdapter) -> None: - # Snapshot this platform slot's current owner first: acting on a stale notification would - # overwrite a healthy platform's runtime status and wrongly re-queue it for reconnection. + # Snapshot the slot owner first: a stale notification must not overwrite a healthy + # platform's runtime status or re-queue it. existing = self.adapters.get(adapter.platform) if existing is not None and existing is not adapter: logger.debug( "Ignoring stale fatal error from a superseded %s adapter instance: %s", - adapter.platform.value, - adapter.fatal_error_code or "unknown", + adapter.platform.value, adapter.fatal_error_code or "unknown", ) return logger.error( - "Fatal %s adapter error (%s): %s", - adapter.platform.value, - adapter.fatal_error_code or "unknown", - adapter.fatal_error_message or "unknown error", + "Fatal %s adapter error (%s): %s", adapter.platform.value, + adapter.fatal_error_code or "unknown", adapter.fatal_error_message or "unknown error", ) - # A relay credential revoked by opt-out is not an error to retry: render a clean "disabled" - # state, not red "fatal"/"retrying" (non-retryable code, so it also leaves the queue below). - if adapter.fatal_error_code == "relay_disabled": - platform_state = "disabled" - elif adapter.fatal_error_retryable: - platform_state = "retrying" - else: - platform_state = "fatal" + # relay_disabled (credential revoked by opt-out) renders "disabled", not red fatal/retrying. self._update_platform_runtime_status( adapter.platform.value, - platform_state=platform_state, + platform_state=( + "disabled" if adapter.fatal_error_code == "relay_disabled" + else "retrying" if adapter.fatal_error_retryable else "fatal" + ), error_code=adapter.fatal_error_code, error_message=adapter.fatal_error_message, ) if existing is adapter: - # Claim this adapter for teardown before awaiting disconnect(): a second fatal-error - # notification for the same adapter (e.g. a concurrent recovery path) would otherwise - # still see itself as "existing" during the await and disconnect() the same object twice. + # Claim for teardown BEFORE awaiting disconnect(), else a concurrent second fatal + # notification for the same adapter would disconnect() it twice. self.adapters.pop(adapter.platform, None) self.delivery_router.adapters = self.adapters - # Queue retryable failures BEFORE any disconnect await: a half-dead transport can wedge - # native close() (or swallow CancelledError), so "disconnect then queue" left platforms - # permanently deaf in a live process after the network recovered. Populate the queue first so the - # reconnect watcher always has work; teardown is best-effort after. + # Queue BEFORE any disconnect await: a half-dead transport can wedge native close() (or + # swallow CancelledError), and "disconnect then queue" left platforms permanently deaf. self._queue_retryable_fatal_platform(adapter) if existing is adapter: - # A half-closed transport can wedge native close() indefinitely; reuse the shutdown-path - # timeout so this runtime fatal handler always returns to the stay-alive / stranded path. + # Bounded by the shutdown-path timeout so this always returns to the stranded check. await self._safe_adapter_disconnect(adapter, adapter.platform) if not self.adapters and not self._failed_platforms: @@ -437,16 +324,23 @@ class GatewayAdapterLifecycleMixin: logger.error("No connected messaging platforms remain. Shutting down gateway cleanly.") await self.stop() elif not self.adapters and self._failed_platforms: - # All platforms are down and queued for reconnection. Keep the gateway alive so cron jobs - # still run and the watcher can recover platforms when the problem clears; exiting for a - # systemd restart would turn a transient outage into a state-killing restart loop. + # Everything is down but queued: stay alive (cron still runs, watcher recovers) rather + # than turn a transient outage into a state-killing systemd restart loop. logger.warning( "No connected messaging platforms remain, but %d platform(s) " "queued for reconnection — gateway staying alive, watcher will " - "retry in background.", - len(self._failed_platforms), + "retry in background.", len(self._failed_platforms), ) + def _retain_background_task(self, task: "asyncio.Task") -> "asyncio.Task": + """Register ``task`` in ``_background_tasks`` (created lazily for bare test runners).""" + tasks = getattr(self, "_background_tasks", None) + if not isinstance(tasks, set): + tasks = self._background_tasks = set() + tasks.add(task) + task.add_done_callback(tasks.discard) + return task + def _request_clean_exit(self, reason: str) -> None: self._exit_cleanly = True self._exit_reason = reason @@ -454,45 +348,34 @@ class GatewayAdapterLifecycleMixin: @staticmethod def _supervised_backoff(attempt: int) -> float: - """Delay before the supervisor's next respawn, in seconds (capped exponential). - - A method so tests can collapse the schedule instead of sleeping through the real curve. - """ + """Capped exponential respawn delay (a method so tests can collapse the schedule).""" return min(60, 2 ** min(attempt, 6)) def _spawn_supervised( - self, coro_factory, name, *, restart=True, _attempt=0, on_spawn=None, - on_give_up=None, + self, coro_factory, name, *, restart=True, _attempt=0, on_spawn=None, on_give_up=None ): """Launch a long-lived background task with task-level supervision. - Catches what a per-iteration try/except cannot — exceptions in the OUTER loop or pre-try - setup — which a bare ``asyncio.create_task`` drops silently. Restarts with capped backoff up - to ``_MAX_SUPERVISED_RESTARTS`` rapid failures; the counter resets after a run healthy for - ``_SUPERVISED_HEALTHY_SECS``. Each spawn uses a fresh ``Context``: an inherited - delegated-child marker would make the Kanban dispatcher reject its own writes. - ``on_spawn`` fires on EVERY spawn incl. respawns; callers tracking the handle elsewhere - (e.g. ``_reconnect_watcher_task``) MUST pass it or a respawn leaves a stale handle and a - SECOND watcher. ``on_give_up(name)`` fires when the restart budget is spent. + Catches outer-loop/pre-try exceptions a bare ``create_task`` drops silently. Restarts with + capped backoff up to ``_MAX_SUPERVISED_RESTARTS`` rapid failures; the counter resets after + ``_SUPERVISED_HEALTHY_SECS`` of healthy running. Each spawn uses a fresh ``Context`` (an + inherited delegated-child marker would make the Kanban dispatcher reject its own writes). + ``on_spawn`` fires on EVERY spawn incl. respawns — callers tracking the handle elsewhere + MUST pass it or a respawn leaves a stale handle and a SECOND watcher. ``on_give_up(name)`` + fires when the restart budget is spent. """ - if getattr(self, "_background_tasks", None) is None: - self._background_tasks = set() - - # Monotonic spawn timestamp captured per spawn: the ``_done`` callback - # uses it to distinguish a rapid crash-loop from a healthy-run-then-crash. + # Spawn timestamp lets ``_done`` tell a rapid crash-loop from a healthy-run-then-crash. _started = time.monotonic() - # Deliberately no kwargs to create_task (some test doubles mock a narrow signature); calling - # it from a fresh Context gives the same isolation as create_task(..., context=Context()). + # No create_task kwargs (test doubles mock a narrow signature); running it from a fresh + # Context gives the same isolation as create_task(..., context=Context()). task = Context().run(lambda: asyncio.create_task(coro_factory())) - # PERMANENT supervised watcher, not transient background WORK: the scale-to-zero idle check - # must ignore process-lifetime watchers or the gateway counts itself busy forever. Transient - # tasks added to _background_tasks elsewhere (startup-resume events etc.) stay counted. + # Marks a PERMANENT watcher so the scale-to-zero idle check ignores it (else the gateway + # counts itself busy forever); transient _background_tasks stay counted. task._hermes_supervised_watcher = True # type: ignore[attr-defined] - self._background_tasks.add(task) + self._retain_background_task(task) if on_spawn is not None: - # Record the live handle NOW so an external tracker (e.g. _reconnect_watcher_task) - # points at the current task, not a dead one left by a prior supervised respawn. + # Record the live handle NOW so external trackers don't point at a dead prior task. try: on_spawn(task) except Exception: # pragma: no cover - defensive; a tracker must never kill the spawn @@ -504,106 +387,74 @@ class GatewayAdapterLifecycleMixin: return exc = t.exception() if exc is None: - # Clean return == deliberate shutdown or a self-disabling watcher (e.g. a gated - # no-op returning at once); respawning would busy-spin it — NEVER restart on it. + # Clean return = deliberate shutdown or a self-disabling watcher; NEVER respawn. return logger.error("Supervised task %s died: %r", name, exc, exc_info=exc) - if restart and self._running: - ran_for = time.monotonic() - _started - if ran_for >= self._SUPERVISED_HEALTHY_SECS: - # Ran healthily before crashing — a FRESH failure, not a rapid crash-loop. Reset - # the counter so a daemon crashing a few times over days is never abandoned. - effective_attempt = 0 - else: - effective_attempt = _attempt - if effective_attempt >= self._MAX_SUPERVISED_RESTARTS: - logger.error( - "Supervised task %s died %d times in rapid succession " - "(each within %ds of restart) — giving up restarts", - name, - effective_attempt, - self._SUPERVISED_HEALTHY_SECS, + if not (restart and self._running): + return + # A healthy run before the crash is a FRESH failure, not a crash-loop: reset so a + # daemon crashing a few times over days is never abandoned. + healthy = time.monotonic() - _started >= self._SUPERVISED_HEALTHY_SECS + effective_attempt = 0 if healthy else _attempt + if effective_attempt >= self._MAX_SUPERVISED_RESTARTS: + logger.error( + "Supervised task %s died %d times in rapid succession " + "(each within %ds of restart) — giving up restarts", name, + effective_attempt, self._SUPERVISED_HEALTHY_SECS, + ) + if on_give_up is not None: + try: + on_give_up(name) + except Exception: # pragma: no cover - defensive + logger.debug("on_give_up callback for %s raised", name, exc_info=True) + return + backoff = self._supervised_backoff(effective_attempt) + + async def _respawn(): + await asyncio.sleep(backoff) + if self._running: + self._spawn_supervised( + coro_factory, name, restart=restart, _attempt=effective_attempt + 1, + on_spawn=on_spawn, on_give_up=on_give_up, # only the LAST give-up matters ) - if on_give_up is not None: - try: - on_give_up(name) - except Exception: # pragma: no cover - defensive - logger.debug( - "on_give_up callback for %s raised", - name, exc_info=True, - ) - return - backoff = self._supervised_backoff(effective_attempt) - async def _respawn(): - await asyncio.sleep(backoff) - if self._running: - self._spawn_supervised( - coro_factory, - name, - restart=restart, - _attempt=effective_attempt + 1, - on_spawn=on_spawn, - # Threaded through the recursion like on_spawn: only the LAST respawn's give-up - # matters, and dropping the callback leaves the exhaustion branch with no owner. - on_give_up=on_give_up, - ) - - # The done callback retains its registration context, so isolate the backoff task - # too; otherwise a restart could reintroduce the original caller's turn scope. - respawn_task = Context().run(lambda: asyncio.create_task(_respawn())) - self._background_tasks.add(respawn_task) - respawn_task.add_done_callback(self._background_tasks.discard) + # The done callback runs in its registration context; isolate the backoff task too. + self._retain_background_task(Context().run(lambda: asyncio.create_task(_respawn()))) task.add_done_callback(_done) return task - async def _handoff_watcher( - self, interval: float = 2.0, drain_timeout: float = 30.0, - ) -> None: - """Background task that processes pending CLI→gateway session handoffs. + async def _handoff_watcher(self, interval: float = 2.0, drain_timeout: float = 30.0) -> None: + """Process pending CLI→gateway session handoffs from ``state.db``. - Polls ``state.db`` for ``handoff_state='pending'`` rows: claim atomically (pending → - running), re-bind the home channel's session_key to the CLI session_id via - ``switch_session``, dispatch a synthetic ``MessageEvent``, mark ``completed``/``failed``. + Claim atomically (pending → running), re-bind the home channel to the CLI session_id, + dispatch a synthetic ``MessageEvent``, mark ``completed``/``failed``. """ from gateway.run import _async_profile_runtime_scope, _handoff_watch_scopes, _reclaim_stale - # Initial delay so the gateway is fully connected to its platforms - # before we try to dispatch handoffs through them. - await asyncio.sleep(5) + await asyncio.sleep(5) # let platforms connect before dispatching through them - # Does _process_handoff accept the profile argument? The real one does; test stand-ins bind - # a one-parameter callable. Probed once, outside the loop. + # Does _process_handoff accept the profile argument? Test stand-ins bind a one-arg callable. try: import inspect as _inspect - _process_takes_profile = len( - _inspect.signature(self._process_handoff).parameters - ) >= 2 + _process_takes_profile = len(_inspect.signature(self._process_handoff).parameters) >= 2 except Exception: _process_takes_profile = False - # In-flight dispatches keyed by session id. A handoff runs a FULL agent turn plus delivery - # (far longer than the CLI's 60s wait); inline processing would let one slow handoff block - # every other profile's poll and time them out. Fire-and-forget; the poll loop only claims. + # In-flight dispatches keyed by session id: a handoff runs a FULL agent turn, so inline + # processing would let one slow handoff time out every other profile's poll. inflight: Dict[str, "asyncio.Task"] = {} async def _dispatch(row, session_id, session_db, profile_name) -> None: """Run one claimed handoff to a terminal state, off the poll path.""" try: - if _process_takes_profile: - await self._process_handoff(row, profile_name) - else: - await self._process_handoff(row) + await self._process_handoff(*((row, profile_name) if _process_takes_profile else (row,))) await session_db.complete_handoff(session_id) except asyncio.CancelledError: # Gateway shutting down: leave the row 'running' so the next # start's reclaim marks it failed with a clear reason. raise except Exception as exc: - logger.warning( - "Handoff for session %s failed: %s", - session_id, exc, exc_info=True, - ) + logger.warning("Handoff for session %s failed: %s", session_id, exc, exc_info=True) try: await session_db.fail_handoff(session_id, str(exc)) except Exception: @@ -612,12 +463,11 @@ class GatewayAdapterLifecycleMixin: inflight.pop(session_id, None) async def _tick(profile_name: Optional[str] = None) -> None: - """One poll of the CURRENTLY-SCOPED session store. + """One poll of the CURRENTLY-SCOPED store; ``profile_name`` (None = root) routes + delivery to that profile's OWN adapter. - A closure over ``self``, not a method: unit tests bind ``_handoff_watcher`` onto a - ``SimpleNamespace`` exposing only ``_session_db``, ``_running`` and ``_process_handoff``; - any other ``self.`` would raise, be swallowed by the loop, and silently no-op the - watcher. ``profile_name`` (``None`` = root) makes delivery use that profile's OWN adapter. + A closure, not a method: tests bind ``_handoff_watcher`` onto a ``SimpleNamespace`` + with only ``_session_db``/``_running``/``_process_handoff``. """ session_db = getattr(self, "_session_db", None) if session_db is None: @@ -630,8 +480,6 @@ class GatewayAdapterLifecycleMixin: if not await session_db.claim_handoff(session_id): # Another tick or another gateway already claimed it. continue - # Positional, not keyword: tests bind a one-arg ``_process_handoff(row)`` stand-in and a - # keyword call would TypeError into the failure branch (arity probed above). # INVARIANT (do not weaken): this task is created inside _profile_runtime_scope but # typically RUNS after it exits; it sees the profile's home/secret scope only because # those seams are ContextVar-based and ensure_future copies the Context. @@ -639,15 +487,17 @@ class GatewayAdapterLifecycleMixin: _dispatch(row, session_id, session_db, profile_name) ) - # A row still 'running' at startup belongs to a gateway that died mid-dispatch: it can never - # reach a terminal state, and request_handoff refuses new requests while it sits there. + # A row still 'running' at startup belongs to a gateway that died mid-dispatch; it blocks + # request_handoff until reclaimed. + def _scope(profile_home): + if profile_home is None: + return contextlib.nullcontext() + return _async_profile_runtime_scope(profile_home) + for _pname, _phome in _handoff_watch_scopes(self): try: - if _phome is None: + async with _scope(_phome): await _reclaim_stale(self) - else: - async with _async_profile_runtime_scope(_phome): - await _reclaim_stale(self) except Exception: logger.debug("Stale-handoff reclaim failed", exc_info=True) @@ -655,19 +505,15 @@ class GatewayAdapterLifecycleMixin: while self._running: try: for profile_name, profile_home in _handoff_watch_scopes(self): - if profile_home is None: + async with _scope(profile_home): await _tick(profile_name) - else: - async with _async_profile_runtime_scope(profile_home): - await _tick(profile_name) except asyncio.CancelledError: raise except Exception as exc: logger.debug("Handoff watcher tick error: %s", exc, exc_info=True) await asyncio.sleep(interval) finally: - # Drain in-flight dispatches before returning: cancelling would strand their rows in - # 'running'; a bounded grace period lets an almost-done handoff record its own state. + # Bounded drain: cancelling would strand in-flight rows in 'running'. pending_tasks = [t for t in inflight.values() if not t.done()] if pending_tasks: try: @@ -679,25 +525,19 @@ class GatewayAdapterLifecycleMixin: task.cancel() def _on_reconnect_watcher_gave_up(self, name: str = "") -> None: - """Own the reconnect invariant once supervision has abandoned it. - - Invariant: while running and ``_failed_platforms`` is non-empty, a reconnect watcher is live - or a bounded respawn is scheduled. Event-coupled recovery is not enough: the failed adapter - is dropped from the live map, so no later event may ever arrive to notice a dead watcher. - Deliberately NOT done here: requesting a process restart when the slow tier is exhausted — - a blast-radius policy call; a single loud error names the still-queued platforms instead. - """ + """Own the reconnect invariant once supervision gives up: while running with queued + platforms, a watcher is live or a bounded respawn is scheduled (no later event can notice + a dead watcher). Slow-tier exhaustion logs loudly; deliberately NOT a process restart.""" if not getattr(self, "_running", False): return - if not getattr(self, "_failed_platforms", None): - # No queued work depends on the watcher; leaving it dead is correct — the enqueue path - # spawns a fresh one the moment a platform is queued again. + if getattr(self, "_failed_platforms", None): + self._schedule_slow_reconnect_watcher_respawn(attempt=0) + else: + # Nothing depends on the watcher; the enqueue path spawns a fresh one when needed. logger.warning( "Reconnect watcher supervision exhausted with an empty retry " "queue — leaving it down until a platform is queued." ) - return - self._schedule_slow_reconnect_watcher_respawn(attempt=0) def _schedule_slow_reconnect_watcher_respawn(self, *, attempt: int) -> None: """Bounded slow-tier respawn of the reconnect watcher.""" @@ -705,69 +545,47 @@ class GatewayAdapterLifecycleMixin: logger.error( "Reconnect watcher could not be kept alive after %d slow " "respawns; %d platform(s) remain queued and unattended: %s. " - "Manual intervention or a gateway restart is required.", - attempt, - len(self._failed_platforms), - ", ".join(str(p) for p in self._failed_platforms), + "Manual intervention or a gateway restart is required.", attempt, + len(self._failed_platforms), ", ".join(str(p) for p in self._failed_platforms), ) return async def _slow_respawn() -> None: await asyncio.sleep(self._RECONNECT_WATCHER_SLOW_RETRY_SECS) - if not getattr(self, "_running", False): - return - if not getattr(self, "_failed_platforms", None): - # The queue drained while we waited -- something else healed - # it. Nothing to own any more. - return task = getattr(self, "_reconnect_watcher_task", None) - if task is not None and not task.done(): - return # a watcher came back on its own; stand down + if ( + not getattr(self, "_running", False) + or not getattr(self, "_failed_platforms", None) # queue drained while waiting + or (task is not None and not task.done()) # a watcher came back; stand down + ): + return logger.warning( "Reconnect watcher still down with %d platform(s) queued — " - "slow respawn %d/%d", - len(self._failed_platforms), - attempt + 1, + "slow respawn %d/%d", len(self._failed_platforms), attempt + 1, self._MAX_SLOW_WATCHER_RESPAWNS, ) self._spawn_reconnect_watcher( - on_give_up=lambda _name: self._schedule_slow_reconnect_watcher_respawn( - attempt=attempt + 1 - ) + on_give_up=lambda _name: self._schedule_slow_reconnect_watcher_respawn(attempt=attempt + 1) ) - respawn_task = asyncio.create_task(_slow_respawn()) - if getattr(self, "_background_tasks", None) is None: - self._background_tasks = set() - self._background_tasks.add(respawn_task) - respawn_task.add_done_callback(self._background_tasks.discard) + self._retain_background_task(asyncio.create_task(_slow_respawn())) def _spawn_reconnect_watcher(self, *, on_give_up=None): - """Single place that knows how to launch the reconnect watcher. - - ``on_spawn`` is load-bearing: without it the supervisor's own respawn leaves - ``_reconnect_watcher_task`` at a dead handle and ``_ensure_...`` spawns a second watcher. - """ + """Launch the reconnect watcher. ``on_spawn`` is load-bearing: without it a supervised + respawn leaves ``_reconnect_watcher_task`` dead and ``_ensure_...`` spawns a second one.""" self._reconnect_watcher_task = self._spawn_supervised( - self._platform_reconnect_watcher, - "platform_reconnect_watcher", + self._platform_reconnect_watcher, "platform_reconnect_watcher", on_spawn=lambda t: setattr(self, "_reconnect_watcher_task", t), on_give_up=on_give_up or self._on_reconnect_watcher_gave_up, ) return self._reconnect_watcher_task def _ensure_reconnect_watcher_running(self) -> None: - """Ensure the platform reconnect watcher background task is alive. - - Respawns a dead watcher (exhausted restart budget, unrecoverable exception) so queued - platforms are not stranded. Called on BOTH _queue_retryable_fatal_platform paths: the - re-fatal of an already-queued platform is the only case where the budget can be exhausted. - """ - if not getattr(self, "_running", False): - return + """Respawn a dead reconnect watcher (called on BOTH _queue_retryable_fatal_platform paths: + the re-fatal of an already-queued platform is the only case that exhausts the budget).""" task = getattr(self, "_reconnect_watcher_task", None) - if task is not None and not task.done(): - return # already alive + if not getattr(self, "_running", False) or (task is not None and not task.done()): + return # not running, or already alive logger.warning( "Reconnect watcher task is dead (done=%s) — respawning", task.done() if task is not None else "N/A", @@ -775,248 +593,185 @@ class GatewayAdapterLifecycleMixin: self._spawn_reconnect_watcher() async def _platform_reconnect_watcher(self) -> None: - """Background task that periodically retries connecting failed platforms. + """Periodically retry failed platforms: backoff 30s → 300s cap, retryable failures retry + forever (self-heal), non-retryable drop out. Pausing is manual only (``/platform pause``).""" + async def _idle(seconds: int, until_queued: bool = False) -> bool: + """Sleep in 1s steps; False once the runner stops (or, if asked, once work is queued).""" + for _ in range(seconds): + if not self._running: + return False + if until_queued and self._failed_platforms: + break + await asyncio.sleep(1) + return True - Exponential backoff 30s → 300s cap; retryable failures (network/DNS) retry at the cap - indefinitely so transient outages self-heal, non-retryable (bad auth) drop out immediately. - The circuit breaker (``/platform pause``) is manual only — auto-pausing left bots dead. - """ - from gateway.run import ( - _dispose_unused_adapter, - _platform_has_bot_credential, - _reconnect_backoff, - _reconnect_needs_attention, - ) await asyncio.sleep(10) # initial delay — let startup finish while self._running: if not self._failed_platforms: - # Nothing to reconnect — sleep and check again - for _ in range(30): - if not self._running: - return - if self._failed_platforms: - break - await asyncio.sleep(1) + if not await _idle(30, until_queued=True): + return continue - now = time.monotonic() for platform in list(self._failed_platforms.keys()): if not self._running: return - info = self._failed_platforms.get(platform) - if info is None: - # Removed concurrently (/platform resume, reconnect via another path) between - # the snapshot above and this lookup — not an error, nothing to do this pass. - continue - # Skip paused platforms entirely — they need explicit - # /platform resume to come back. - if info.get("paused"): - continue - # Long-lived retry escalation: past the attention threshold flag the platform - # NEEDS_ATTENTION in runtime status so a dead token/revoked intent doesn't look - # like ordinary "retrying" forever. A signal, NOT a circuit breaker — retries continue. - if not info.get("attention_flagged") and _reconnect_needs_attention(info, now): - info["attention_flagged"] = True - queued_for = now - info.get("queued_at", now) - retrying_since_iso = ( - datetime.now(timezone.utc) - timedelta(seconds=queued_for) - ).isoformat() - logger.warning( - "%s has been failing/reconnecting continuously for " - "%.1f hours (%d attempts) — flagging NEEDS_ATTENTION. " - "Retries continue, but this usually means a permanent " - "problem (revoked credentials, missing intents, broken " - "sidecar). Check `hermes status` / `/platform list`.", - platform.value, - queued_for / 3600.0, - info.get("attempts", 0), - ) - self._update_platform_runtime_status( - platform.value, - platform_state="retrying", - needs_attention=True, - retrying_since=retrying_since_iso, - ) - if now < info["next_retry"]: - continue # not time yet + await self._reconnect_failed_platform(platform, now) + if not await _idle(10): # re-check every 10 seconds + return - platform_config = info["config"] - attempt = info["attempts"] + 1 - # Empty-token primary configs can never reconnect; drop them so multiplex setups - # where a secondary profile owns the bot do not spin forever. - if not _platform_has_bot_credential(platform, platform_config): - logger.warning( - "Reconnect %s: no bot credential on queued config, " - "removing from retry queue", - platform.value, - ) - del self._failed_platforms[platform] - continue - logger.info( - "Reconnecting %s (attempt %d)...", - platform.value, attempt, + def _flag_reconnect_needs_attention(self, platform, info: dict, now: float) -> None: + """Flag NEEDS_ATTENTION (once) past the threshold — a signal, NOT a circuit breaker.""" + from gateway.run import _reconnect_needs_attention + if info.get("attention_flagged") or not _reconnect_needs_attention(info, now): + return + info["attention_flagged"] = True + queued_for = now - info.get("queued_at", now) + retrying_since_iso = (datetime.now(timezone.utc) - timedelta(seconds=queued_for)).isoformat() + logger.warning( + "%s has been failing/reconnecting continuously for " + "%.1f hours (%d attempts) — flagging NEEDS_ATTENTION. " + "Retries continue, but this usually means a permanent " + "problem (revoked credentials, missing intents, broken " + "sidecar). Check `hermes status` / `/platform list`.", platform.value, + queued_for / 3600.0, info.get("attempts", 0), + ) + self._update_platform_runtime_status( + platform.value, platform_state="retrying", needs_attention=True, + retrying_since=retrying_since_iso, + ) + + def _bump_reconnect_backoff( + self, platform, info: dict, attempt: int, error_code, error_message: str + ) -> int: + """Mark the platform retrying and record the failed attempt; returns the backoff applied.""" + from gateway.run import _reconnect_backoff + self._update_platform_runtime_status( + platform.value, platform_state="retrying", error_code=error_code, error_message=error_message, + ) + backoff = _reconnect_backoff(attempt) + info["attempts"] = attempt + info["next_retry"] = time.monotonic() + backoff + return backoff + + async def _reconnect_failed_platform(self, platform, now: float) -> None: + """One watcher pass for a queued platform: gate, attempt, and record the outcome.""" + from gateway.run import _dispose_unused_adapter, _platform_has_bot_credential + info = self._failed_platforms.get(platform) + # None: removed concurrently since the caller's snapshot. Paused needs /platform resume. + if info is None or info.get("paused"): + return + self._flag_reconnect_needs_attention(platform, info, now) + if now < info["next_retry"]: + return # not time yet + + platform_config = info["config"] + attempt = info["attempts"] + 1 + # Empty-token primary configs can never reconnect; drop them so multiplex setups + # where a secondary profile owns the bot do not spin forever. + if not _platform_has_bot_credential(platform, platform_config): + self._drop_from_reconnect_queue(platform, "no bot credential on queued config") + return + logger.info("Reconnecting %s (attempt %d)...", platform.value, attempt) + + adapter = None + try: + adapter = self._create_adapter(platform, platform_config) + if not adapter: + self._drop_from_reconnect_queue(platform, "adapter creation returned None") + return + + self._wire_adapter_handlers(adapter) + # Reconnect after outage: keep the platform's server-side update queue so + # messages sent while the bot was offline are delivered rather than dropped. + success = await self._connect_adapter_with_timeout(adapter, platform, is_reconnect=True) + if success: + await self._install_reconnected_adapter(platform, adapter) + elif adapter.has_fatal_error and not adapter.fatal_error_retryable: + self._update_platform_runtime_status( + platform.value, platform_state="fatal", error_code=adapter.fatal_error_code, + error_message=adapter.fatal_error_message, ) + logger.warning( + "Reconnect %s: non-retryable error (%s), removing from retry queue", + platform.value, adapter.fatal_error_message, + ) + # Never installed on self.adapters, so nothing else will disconnect() it: dispose or + # the resource owners built in __init__ leak ~2 fds each (fd limit in ~12h at the cap). + await _dispose_unused_adapter(adapter) + del self._failed_platforms[platform] + else: + # Retryable failures retry at the backoff cap forever; never auto-pause them — a + # transient outage must not need `/platform resume`. Same fd-leak dispose as above. + backoff = self._bump_reconnect_backoff( + platform, info, attempt, adapter.fatal_error_code, + adapter.fatal_error_message or "failed to reconnect", + ) + logger.info("Reconnect %s failed, next retry in %ds", platform.value, backoff) + await _dispose_unused_adapter(adapter) + except Exception as e: + if adapter is not None: + # An exception escaping connect leaves the adapter in the same unowned state. + await _dispose_unused_adapter(adapter) + # A reconnect exception is transient; keep retrying at the cap rather than auto-pausing. + backoff = self._bump_reconnect_backoff(platform, info, attempt, None, str(e)) + logger.warning("Reconnect %s error: %s, next retry in %ds", platform.value, e, backoff) - adapter = None - try: - adapter = self._create_adapter(platform, platform_config) - if not adapter: - logger.warning( - "Reconnect %s: adapter creation returned None, removing from retry queue", - platform.value, - ) - del self._failed_platforms[platform] - continue + def _drop_from_reconnect_queue(self, platform, reason: str) -> None: + logger.warning("Reconnect %s: %s, removing from retry queue", platform.value, reason) + del self._failed_platforms[platform] - adapter.set_message_handler(self._primary_message_handler()) - adapter.set_fatal_error_handler(self._handle_adapter_fatal_error) - adapter.set_session_store(self.session_store) - adapter.set_busy_session_handler(self._handle_active_session_busy_message) - _set_reaction = getattr(adapter, "set_reaction_handler", None) - if callable(_set_reaction): - _set_reaction(self._handle_reaction_event) - adapter.set_topic_recovery_fn(self._recover_telegram_topic_thread_id) - adapter.set_authorization_check(self._make_adapter_auth_check(adapter.platform)) - adapter.set_platform_event_handler(self._primary_platform_event_handler()) - adapter._busy_text_mode = self._busy_text_mode + async def _install_reconnected_adapter(self, platform, adapter) -> None: + """Publish a freshly reconnected primary adapter and replay what it missed while down.""" + self.adapters[platform] = adapter + self._sync_voice_mode_state_to_adapter(adapter) + self._bind_voice_input_callback(adapter) + self.delivery_router.adapters = self.adapters + del self._failed_platforms[platform] + self._update_platform_runtime_status( + platform.value, platform_state="connected", error_code=None, error_message=None, + needs_attention=False, retrying_since=None, + ) + logger.info("✓ %s reconnected successfully", platform.value) - # Reconnect after outage: keep the platform's server-side update queue so - # messages sent while the bot was offline are delivered rather than dropped. - success = await self._connect_adapter_with_timeout( - adapter, platform, is_reconnect=True - ) - if success: - self.adapters[platform] = adapter - self._sync_voice_mode_state_to_adapter(adapter) - # Wire voice input callback on reconnect as well (#60623). - self._bind_voice_input_callback(adapter) - self.delivery_router.adapters = self.adapters - del self._failed_platforms[platform] - self._update_platform_runtime_status( - platform.value, - platform_state="connected", - error_code=None, - error_message=None, - needs_attention=False, - retrying_since=None, - ) - logger.info("✓ %s reconnected successfully", platform.value) + # Final responses rejected while this adapter was down are still owned by this live + # process, so startup recovery cannot claim them. Replay the transient subset now. + try: + await self._redeliver_failed_obligations_for_platform(platform) + except Exception: + logger.debug( + "failed-obligation redelivery after %s reconnect failed", platform.value, exc_info=True, + ) - # Final responses rejected while this adapter was down are still owned by - # this live process, so startup recovery cannot claim them. Replay the - # explicitly transient subset now that the platform is usable. - try: - await self._redeliver_failed_obligations_for_platform( - platform - ) - except Exception: - logger.debug( - "failed-obligation redelivery after %s reconnect failed", - platform.value, - exc_info=True, - ) + # Rebuild channel directory with the new adapter + try: + from gateway.channel_directory import build_channel_directory + await build_channel_directory(self.adapters) + except Exception: + pass - # Rebuild channel directory with the new adapter - try: - from gateway.channel_directory import build_channel_directory - await build_channel_directory(self.adapters) - except Exception: - pass - - # A platform that was offline at gateway startup never got its restart- - # interrupted sessions auto-resumed — the startup pass skips sessions whose - # adapter isn't connected yet. - try: - self._schedule_resume_pending_sessions(platform=platform) - except Exception: - logger.debug( - "resume-pending reschedule after %s reconnect failed", - platform.value, - exc_info=True, - ) - # Check if the failure is non-retryable - elif adapter.has_fatal_error and not adapter.fatal_error_retryable: - self._update_platform_runtime_status( - platform.value, - platform_state="fatal", - error_code=adapter.fatal_error_code, - error_message=adapter.fatal_error_message, - ) - logger.warning( - "Reconnect %s: non-retryable error (%s), removing from retry queue", - platform.value, adapter.fatal_error_message, - ) - # The adapter is about to be dropped from the queue without ever being - # installed on self.adapters, so nothing else will call disconnect() on it. - # Dispose here or the resource owners built in __init__ (ResponseStore etc.) - # leak ~2 fds each; at the 300s cap the gateway hits the fd limit in ~12h. - await _dispose_unused_adapter(adapter) - del self._failed_platforms[platform] - else: - self._update_platform_runtime_status( - platform.value, - platform_state="retrying", - error_code=adapter.fatal_error_code, - error_message=adapter.fatal_error_message or "failed to reconnect", - ) - backoff = _reconnect_backoff(attempt) - info["attempts"] = attempt - info["next_retry"] = time.monotonic() + backoff - logger.info( - "Reconnect %s failed, next retry in %ds", - platform.value, backoff, - ) - # Same fd-leak concern as the non-retryable branch above: the adapter failed - # to connect and is being thrown away. - await _dispose_unused_adapter(adapter) - # Retryable failures (network/DNS blips) retry at the backoff cap forever, - # self-healing when connectivity returns. Never auto-pause them: a transient - # outage must not need `/platform resume`. Everything here is retryable. - except Exception as e: - if adapter is not None: - # An exception escaping connect (DNS timeout, aiohttp server.start() crash, - # etc.) leaves the adapter in the same unowned state as the branches above. - await _dispose_unused_adapter(adapter) - self._update_platform_runtime_status( - platform.value, - platform_state="retrying", - error_code=None, - error_message=str(e), - ) - backoff = _reconnect_backoff(attempt) - info["attempts"] = attempt - info["next_retry"] = time.monotonic() + backoff - logger.warning( - "Reconnect %s error: %s, next retry in %ds", - platform.value, e, backoff, - ) - # A reconnect exception (connect timeout, DNS failure, ...) is transient; keep - # retrying at the backoff cap rather than auto-pausing. - - # Check every 10 seconds for platforms that need reconnection - for _ in range(10): - if not self._running: - return - await asyncio.sleep(1) + # A platform offline at gateway startup never got its restart-interrupted sessions + # auto-resumed — the startup pass skips sessions whose adapter isn't connected yet. + try: + self._schedule_resume_pending_sessions(platform=platform) + except Exception: + logger.debug( + "resume-pending reschedule after %s reconnect failed", platform.value, exc_info=True + ) async def _cancel_secondary_profile_reconnect_tasks(self) -> None: - """Cancel profile-scoped reconnects before tearing down their registry. - - A reconnect can be waiting in adapter setup while shutdown begins. It must not republish - an adapter after the secondary registry is drained. Waiting is bounded by the adapter- - cleanup budget; a task that overruns is still blocked by the stopped runner state. - """ + """Cancel profile-scoped reconnects before tearing down their registry, so a reconnect + mid-setup cannot republish an adapter after the registry drains (bounded wait).""" pending = self._profile_failed_platforms if not isinstance(pending, dict): return current = asyncio.current_task() - tasks: list[asyncio.Task] = [] - for profile_pending in pending.values(): - if not isinstance(profile_pending, dict): - continue - for task in profile_pending.values(): - if isinstance(task, asyncio.Task) and task is not current and not task.done(): - tasks.append(task) + tasks = [ + task + for profile_pending in pending.values() + if isinstance(profile_pending, dict) + for task in profile_pending.values() + if isinstance(task, asyncio.Task) and task is not current and not task.done() + ] for task in tasks: task.cancel() timeout = self._adapter_disconnect_timeout_secs() @@ -1030,18 +785,13 @@ class GatewayAdapterLifecycleMixin: pending.clear() async def _start_secondary_profile_adapters(self) -> int: - """Bring up adapters for every non-active profile this gateway serves. + """Bring up adapters for every non-active profile (multiplex only); returns connected count. - Returns the count of connected secondary adapters; 0 unless ``gateway.multiplex_profiles``. - Each profile's adapters connect under its HERMES_HOME + secret scope, live in - ``self._profile_adapters[profile]``, and get a handler stamping ``source.profile``. Same- - platform credential collisions are refused here — the only point seeing every profile's - resolved credentials together. + Each profile connects under its own HERMES_HOME + secret scope; credential/listener + collisions are refused here — the only point seeing every profile's credentials together. """ from gateway.run import ( - MultiplexConfigError, - SecondaryPortBindingConfigError, - _multiplex_profile_homes, + MultiplexConfigError, SecondaryPortBindingConfigError, _multiplex_profile_homes ) if not getattr(self.config, "multiplex_profiles", False): return 0 @@ -1053,117 +803,94 @@ class GatewayAdapterLifecycleMixin: active = get_active_profile_name() or "default" connected = 0 - # Resource claim -> owning profile. Credential claims stop two profiles polling the same - # account; listener claims stop sidecars with distinct credentials binding one endpoint. - claimed: Dict[tuple, str] = {} - for _plat, _ad in self.adapters.items(): - fp = self._adapter_credential_fingerprint(_ad) - if fp is not None: - claimed[(_plat, fp)] = active - listener_claim = self._adapter_listener_claim(_plat, _ad) - if listener_claim is not None: - claimed[listener_claim] = active - # A retryable primary still owns its credential and listener; reserve both while queued - # so a secondary cannot take the endpoint before the reconnect watcher retries it. - for retry_info in getattr(self, "_failed_platforms", {}).values(): - for claim_name in ("credential_claim", "listener_claim"): - retry_claim = retry_info.get(claim_name) - if isinstance(retry_claim, tuple): - claimed[retry_claim] = active - + claimed = self._primary_resource_claims(active) profile_homes = _multiplex_profile_homes(self.config) for profile_name, profile_home in profile_homes: if profile_name == active: continue # handled by the primary startup loop try: - connected += await self._start_one_profile_adapters( - profile_name, profile_home, claimed - ) + connected += await self._start_one_profile_adapters(profile_name, profile_home, claimed) except SecondaryPortBindingConfigError as e: logger.warning( - "Skipping secondary profile '%s' due to port-binding config error: %s", - profile_name, - e, + "Skipping secondary profile '%s' due to port-binding config error: %s", profile_name, e, ) except MultiplexConfigError: raise except Exception as e: logger.error( - "Failed to start adapters for profile '%s': %s", - profile_name, e, exc_info=True, + "Failed to start adapters for profile '%s': %s", profile_name, e, exc_info=True ) - # Record the authoritative served set in runtime status for `hermes status`. "Served" - # means eligible for shared routing, HTTP prefixes, cron, and profile runtime scope — - # intentionally broader than profiles with a connected (or any) secondary adapter. + self._record_served_profiles(active, profile_homes) + return connected + + def _primary_resource_claims(self, active: str) -> Dict[tuple, str]: + """Resource claim -> owning profile for every live or queued primary adapter (credential: + one account polled once; listener: one bind+port). A queued retryable primary owns both.""" + claimed: Dict[tuple, str] = {} + for _plat, _ad in self.adapters.items(): + fp = self._adapter_credential_fingerprint(_ad) + for claim in ((_plat, fp) if fp is not None else None, self._adapter_listener_claim(_plat, _ad)): + if claim is not None: + claimed[claim] = active + for retry_info in getattr(self, "_failed_platforms", {}).values(): + for claim_name in ("credential_claim", "listener_claim"): + retry_claim = retry_info.get(claim_name) + if isinstance(retry_claim, tuple): + claimed[retry_claim] = active + return claimed + + def _record_served_profiles(self, active: str, profile_homes) -> None: + """Record the served set (eligible for routing/HTTP prefixes/cron/runtime scope — broader + than "has a connected adapter") for `hermes status`; seed per-profile PairingStores.""" try: from gateway.status import write_runtime_status from gateway.pairing import PairingStore - served = [active] + sorted( - name for name, _home in profile_homes if name != active - ) - # Per-profile PairingStores so authz_mixin routes pairing checks to the right whitelist; - # the active profile's store is at its HERMES_HOME, other served profiles at their own. + served = [active] + sorted(name for name, _home in profile_homes if name != active) for name in served: if name and name not in self.pairing_stores: self.pairing_stores[name] = ( - self.pairing_store - if name == active - else PairingStore(profile=name) + self.pairing_store if name == active else PairingStore(profile=name) ) write_runtime_status(served_profiles=served) except Exception: logger.debug("could not record served_profiles", exc_info=True) - return connected + async def _load_secondary_profile_config(self, profile_name: str, profile_home: "Path"): + """Hydrate + enter ``profile_home``'s scope once; return its gateway config. - async def _start_one_profile_adapters( - self, profile_name: str, profile_home: "Path", claimed: Dict[tuple, str] - ) -> int: - """Create+connect one profile's adapters under its runtime scope.""" + Raises ``MultiplexConfigError`` (open dm/group policy) or ``SecondaryPortBindingConfigError`` + (the default profile owns the single shared HTTP listener). + """ from gateway.run import ( - MultiplexConfigError, - SecondaryPortBindingConfigError, - _load_gateway_runtime_config, - _own_policy_open_startup_violation, - _platform_has_bot_credential, - _profile_runtime_scope, + MultiplexConfigError, SecondaryPortBindingConfigError, _load_gateway_runtime_config, + _own_policy_open_startup_violation, _profile_runtime_scope, ) from gateway.config import load_gateway_config from hermes_cli.env_loader import hydrate_profile_secret_sources - # Hydrate external secret sources (1Password/vault/...) off-loop ONCE, then enter the scope - # without re-hydrating: the sync hydration is network-bound and would otherwise stall every - # other profile's heartbeat while this one boots (same class as the reconnect path). + # Hydrate external secret sources (1Password/vault/...) off-loop ONCE: the sync hydration is + # network-bound and would stall every other profile's heartbeat while this one boots. await asyncio.to_thread(hydrate_profile_secret_sources, profile_home) with _profile_runtime_scope(profile_home, hydrate_secrets=False): profile_runtime_cfg = _load_gateway_runtime_config() from hermes_cli.plugins import discover_plugins - discover_plugins() - # Register this profile's own declarative shell hooks and outbound webhooks. The - # registration in start() runs before any profile scope exists and only sees the root - # profile's config, so without this a secondary profile's `hooks:` block is silently - # inert (its turns use a plugin manager keyed by resolved home). + # Register this profile's shell hooks / outbound webhooks: start() registers before any + # profile scope exists, so a secondary profile's `hooks:` block would be silently inert. try: from hermes_cli.config import load_config as _load_profile_config - from agent.shell_hooks import ( - register_from_config as _register_shell_hooks, - ) - from agent.outbound_webhooks import ( - register_from_config as _register_outbound_webhooks, - ) + from agent.shell_hooks import register_from_config as _register_shell_hooks + from agent.outbound_webhooks import (register_from_config as _register_outbound_webhooks) _profile_hooks_cfg = _load_profile_config() _register_shell_hooks(_profile_hooks_cfg, accept_hooks=False) _register_outbound_webhooks(_profile_hooks_cfg) except Exception: logger.warning( - "shell-hook/webhook registration failed for profile '%s'", - profile_name, - exc_info=True, + "shell-hook/webhook registration failed for profile '%s'", profile_name, exc_info=True, ) profile_cfg = load_gateway_config() @@ -1180,286 +907,252 @@ class GatewayAdapterLifecycleMixin: port_binding_platforms = sorted( platform.value for platform, platform_config in profile_cfg.platforms.items() - if platform_config.enabled - and _platform_binds_port(platform.value, platform_config.extra) + if platform_config.enabled and _platform_binds_port(platform.value, platform_config.extra) ) if port_binding_platforms: - joined = ", ".join(port_binding_platforms) raise SecondaryPortBindingConfigError( f"Profile '{profile_name}' enables port-binding platform(s) " - f"{joined}, but gateway.multiplex_profiles is on. The default " + f"{', '.join(port_binding_platforms)}, but gateway.multiplex_profiles is on. The default " f"profile owns the single shared HTTP listener and serves every " f"profile through the /p/{profile_name}/ URL prefix. Remove " f"these platform entries from profile '{profile_name}'s config.yaml " f"or configure them only on the default profile." ) + return profile_cfg + def _refuse_duplicate_claim( + self, claim, claimed: Dict[tuple, str], profile_name: str, platform: Platform, kind: str + ) -> bool: + """Log + park a secondary adapter whose credential/listener another profile owns (True when + refused). NOT disconnected: it never connected, and for a same-credential Photon adapter + disconnect() would shut down the primary profile's live sidecar.""" + owner = claimed.get(claim) if claim is not None else None + if owner is None: + return False + if kind == "credential": + message = ( + f"Profile '{owner}' and '{profile_name}' both configure " + f"{platform.value} with the same credential. Give each " + f"profile its own {platform.value} credential." + ) + logger.error( + "Profile '%s' and '%s' both configure %s with the same " + "credential — refusing to start the duplicate (one " + "credential cannot be consumed twice). Give each profile " + "its own %s credential.", owner, profile_name, platform.value, platform.value, + ) + else: + bind, port = claim[-2:] + message = ( + f"Profile '{owner}' and '{profile_name}' both configure " + f"{platform.value} sidecars on the same listener. Configure " + f"a distinct listener for profile '{profile_name}'." + ) + logger.error( + "Profile '%s' and '%s' both configure %s sidecars on " + "%s:%s — refusing to start the duplicate listener. " + "Set platforms.%s.extra.sidecar_port to a distinct port " + "for profile '%s'.", + owner, profile_name, platform.value, bind, port, platform.value, profile_name, + ) + self._update_platform_runtime_status( + f"{profile_name}:{platform.value}", platform_state="fatal", + error_code=f"duplicate_{kind}", error_message=message, + ) + return True + + async def _start_one_profile_adapters( + self, profile_name: str, profile_home: "Path", claimed: Dict[tuple, str] + ) -> int: + """Create+connect one profile's adapters under its runtime scope.""" + from gateway.run import _platform_has_bot_credential, _profile_runtime_scope + profile_cfg = await self._load_secondary_profile_config(profile_name, profile_home) + multiplex = getattr(self.config, "multiplex_profiles", False) profile_map = self._profile_adapters.setdefault(profile_name, {}) connected = 0 for platform, platform_config in profile_cfg.platforms.items(): if not platform_config.enabled: continue - # A platform enabled in a secondary profile's config.yaml may have no credential in that - # profile's secret scope — the shared YAML enables it for the default profile only. - # Building an adapter anyway would fan one inbound message out across every - # credential-less profile; mirror the primary loop's credential gate and skip. - if ( - getattr(self.config, "multiplex_profiles", False) - and not _platform_has_bot_credential(platform, platform_config) - ): + # Enabled in the shared YAML but no credential in THIS profile's scope: building an + # adapter anyway would fan one inbound message out across every credential-less profile. + if multiplex and not _platform_has_bot_credential(platform, platform_config): logger.info( "[MULTIPLEX] Profile '%s': skipping %s - no bot credential " - "in this profile's secrets", - profile_name, - platform.value, + "in this profile's secrets", profile_name, platform.value, ) continue - # Relay and WhatsApp are shared process-level ingress in multiplex mode (one connection - # owned by the active profile, route-stamped source.profile fans out). WhatsApp is one - # session per phone number; a secondary adapter would only retry-loop and stall startup. - if ( - getattr(self.config, "multiplex_profiles", False) - and platform in (Platform.RELAY, Platform.WHATSAPP) - ): + # Relay/WhatsApp are shared process-level ingress under multiplex (the active profile's + # one connection fans out by route); a secondary would only retry-loop and stall startup. + if multiplex and platform in (Platform.RELAY, Platform.WHATSAPP): continue try: with _profile_runtime_scope(profile_home, hydrate_secrets=False): adapter = self._create_adapter(platform, platform_config) except Exception as e: logger.error( - "[MULTIPLEX] Profile '%s': _create_adapter('%s') raised %s", - profile_name, - platform.value, - e, - exc_info=True, + "[MULTIPLEX] Profile '%s': _create_adapter('%s') raised %s", profile_name, + platform.value, e, exc_info=True, ) continue if not adapter: logger.warning( "[MULTIPLEX] Profile '%s': skipping platform '%s' - adapter creation returned None", - profile_name, - platform.value, + profile_name, platform.value, ) continue - - # Same-token conflict detection — refuse a duplicate poll. + # Same-token / same-listener conflict detection — refuse a duplicate poll or bind. credential_claim = self._adapter_credential_claim(platform, adapter) - if credential_claim is not None: - owner = claimed.get(credential_claim) - if owner is not None: - message = ( - f"Profile '{owner}' and '{profile_name}' both configure " - f"{platform.value} with the same credential. Give each " - f"profile its own {platform.value} credential." - ) - logger.error( - "Profile '%s' and '%s' both configure %s with the same " - "credential — refusing to start the duplicate (one " - "credential cannot be consumed twice). Give each profile " - "its own %s credential.", - owner, profile_name, platform.value, platform.value, - ) - self._update_platform_runtime_status( - f"{profile_name}:{platform.value}", - platform_state="fatal", - error_code="duplicate_credential", - error_message=message, - ) - # This adapter has not connected and therefore owns no resources to clean up. - # Calling disconnect here can mutate the shared platform state and, for a same- - # credential Photon adapter, shut down the primary profile's live sidecar. - continue - listener_claim = self._adapter_listener_claim(platform, adapter) - if listener_claim is not None: - owner = claimed.get(listener_claim) - if owner is not None: - bind, port = listener_claim[-2:] - message = ( - f"Profile '{owner}' and '{profile_name}' both configure " - f"{platform.value} sidecars on the same listener. Configure " - f"a distinct listener for profile '{profile_name}'." - ) - logger.error( - "Profile '%s' and '%s' both configure %s sidecars on " - "%s:%s — refusing to start the duplicate listener. " - "Set platforms.%s.extra.sidecar_port to a distinct port " - "for profile '%s'.", - owner, - profile_name, - platform.value, - bind, - port, - platform.value, - profile_name, - ) - self._update_platform_runtime_status( - f"{profile_name}:{platform.value}", - platform_state="fatal", - error_code="duplicate_listener", - error_message=message, - ) - # Like credential conflicts, this adapter never connected - # and owns no resources that should be disconnected. - continue - + if self._refuse_duplicate_claim( + credential_claim, claimed, profile_name, platform, "credential" + ) or self._refuse_duplicate_claim(listener_claim, claimed, profile_name, platform, "listener"): + continue self._configure_profile_adapter(adapter, profile_name, platform) - try: with _profile_runtime_scope(profile_home, hydrate_secrets=False): - success = await self._connect_initial_adapter_with_timeout( - adapter, platform - ) - if success: - profile_map[platform] = adapter - # Restore persisted /voice state for this bot (#84872) — - # primary startup and every reconnect path already do. - self._sync_voice_mode_state_to_adapter(adapter) - if credential_claim is not None: - claimed[credential_claim] = profile_name - if listener_claim is not None: - claimed[listener_claim] = profile_name - connected += 1 - logger.info("✓ %s connected (profile: %s)", platform.value, profile_name) - else: + success = await self._connect_initial_adapter_with_timeout(adapter, platform) + if not success: logger.warning("✗ %s failed to connect (profile: %s)", platform.value, profile_name) - await self._safe_adapter_disconnect(adapter, platform) - self._schedule_secondary_profile_startup_reconnect( - profile_name, platform, adapter - ) except Exception as e: logger.error("✗ %s error (profile: %s): %s", platform.value, profile_name, e) + success = False + if not success: await self._safe_adapter_disconnect(adapter, platform) - self._schedule_secondary_profile_startup_reconnect( - profile_name, platform, adapter - ) + self._schedule_secondary_profile_startup_reconnect(profile_name, platform, adapter) + continue + profile_map[platform] = adapter + # Restore persisted /voice state for this bot (primary startup and reconnects do too). + self._sync_voice_mode_state_to_adapter(adapter) + for claim in (credential_claim, listener_claim): + if claim is not None: + claimed[claim] = profile_name + connected += 1 + logger.info("✓ %s connected (profile: %s)", platform.value, profile_name) return connected - def _configure_profile_adapter( - self, - adapter: BasePlatformAdapter, - profile_name: str, - platform: Platform, + def _wire_adapter_handlers( + self, adapter: BasePlatformAdapter, *, message_handler=None, fatal_error_handler=None, + busy_session_handler=None, authorization_check=None, platform_event_handler=None, + busy_text_mode: Optional[str] = None, ) -> None: - """Install the profile-scoped handlers shared by startup and reconnect.""" - # Runtime status is process-scoped while message/config work is profile-scoped. Keep both - # dimensions in the key so dashboard/NAS health aggregation sees which secondary failed. - adapter._runtime_status_platform_key = f"{profile_name}:{platform.value}" - adapter.set_message_handler(self._make_profile_message_handler(profile_name)) - adapter.set_fatal_error_handler( - self._make_profile_fatal_error_handler(profile_name, platform) - ) + """Install the runner callbacks every adapter needs (defaults = primary handlers; + secondary wiring passes profile-scoped variants). ``set_reaction_handler`` is optional.""" + adapter.set_message_handler(message_handler or self._primary_message_handler()) + adapter.set_fatal_error_handler(fatal_error_handler or self._handle_adapter_fatal_error) adapter.set_session_store(self.session_store) - # Declare credential ownership BEFORE any inbound event can be handled: adapter-level - # session keys (batching, _active_sessions, busy guard) are derived at ingress, before the - # handler stamps source.profile — without this every secondary bot would key into the - # default profile's `agent:main:` lane (see BasePlatformAdapter._session_key_profile). - _set_owner = getattr(adapter, "set_owner_profile", None) - if callable(_set_owner): - _set_owner(profile_name) - adapter.set_busy_session_handler( - self._make_profile_busy_session_handler(profile_name) - ) + adapter.set_busy_session_handler(busy_session_handler or self._handle_active_session_busy_message) _set_reaction = getattr(adapter, "set_reaction_handler", None) if callable(_set_reaction): _set_reaction(self._handle_reaction_event) adapter.set_topic_recovery_fn(self._recover_telegram_topic_thread_id) adapter.set_authorization_check( - self._make_adapter_auth_check(platform, profile_name=profile_name) + authorization_check or self._make_adapter_auth_check(adapter.platform) ) - adapter.set_platform_event_handler( - self._make_profile_platform_event_handler(profile_name) - ) - # Voice transcripts from this bot's channels dispatch through THIS - # adapter (primary wiring lives at connect time; see #75198). - self._bind_voice_input_callback(adapter) + adapter.set_platform_event_handler(platform_event_handler or self._primary_platform_event_handler()) + adapter._busy_text_mode = (self._busy_text_mode if busy_text_mode is None else busy_text_mode) + + def _configure_profile_adapter( + self, adapter: BasePlatformAdapter, profile_name: str, platform: Platform + ) -> None: + """Install the profile-scoped handlers shared by startup and reconnect.""" + # Runtime status is process-scoped while message/config work is profile-scoped. Keep both + # dimensions in the key so dashboard/NAS health aggregation sees which secondary failed. + adapter._runtime_status_platform_key = f"{profile_name}:{platform.value}" + # Declare ownership BEFORE any inbound event: adapter-level session keys are derived at + # ingress, before the handler stamps source.profile — without this every secondary bot + # would key into the default profile's `agent:main:` lane. + _set_owner = getattr(adapter, "set_owner_profile", None) + if callable(_set_owner): + _set_owner(profile_name) text_modes = getattr(self, "_busy_text_modes_by_profile", None) - adapter._busy_text_mode = ( - text_modes.get(profile_name, self._busy_text_mode) - if isinstance(text_modes, dict) - else self._busy_text_mode + self._wire_adapter_handlers( + adapter, + message_handler=self._make_profile_message_handler(profile_name), + fatal_error_handler=self._make_profile_fatal_error_handler(profile_name, platform), + busy_session_handler=self._make_profile_busy_session_handler(profile_name), + authorization_check=self._make_adapter_auth_check(platform, profile_name=profile_name), + platform_event_handler=self._make_profile_platform_event_handler(profile_name), + busy_text_mode=( + text_modes.get(profile_name, self._busy_text_mode) + if isinstance(text_modes, dict) + else self._busy_text_mode + ), ) - # Secondary adapters always carry the profile they serve so prune - # paths namespace topic bindings correctly under multiplex (#76423). + # Voice transcripts from this bot's channels dispatch through THIS adapter (primary + # wiring lives at connect time). + self._bind_voice_input_callback(adapter) + # Secondary adapters always carry the profile they serve so prune paths namespace topic + # bindings correctly under multiplex. adapter._hermes_profile_name = profile_name - async def _run_secondary_profile_reconnect( - self, profile_name: str, platform: Platform - ) -> None: + async def _secondary_reconnect_attempt(self, profile_name: str, platform: Platform): + """One scoped attempt to rebuild+connect a secondary adapter → ``(adapter, success)``. + + ``(None, None)`` = give up for good (disabled, credential removed, adapter unavailable). + Caller tears down a RETURNED adapter; one whose configure/connect raised is torn down here. + """ + from gateway.run import _platform_has_bot_credential, _profile_runtime_scope + # Lazy + per-attempt: keeps test monkeypatches on these modules live. + from hermes_cli.profiles import get_profile_dir + from hermes_cli.env_loader import hydrate_profile_secret_sources + from gateway.config import load_gateway_config + profile_home = get_profile_dir(profile_name) + # Hydrate external secret sources off-loop so they cannot starve heartbeats. + await asyncio.to_thread(hydrate_profile_secret_sources, profile_home) + with _profile_runtime_scope(profile_home, hydrate_secrets=False): + profile_config = load_gateway_config().platforms.get(platform) + if profile_config is None or not profile_config.enabled: + return None, None + # Startup credential gate mirror: a removed credential must not rebuild. + if not _platform_has_bot_credential(platform, profile_config): + logger.info( + "Secondary %s reconnect skipped: no bot credential (profile: %s)", + platform.value, profile_name, + ) + return None, None + adapter = self._create_adapter(platform, profile_config) + if adapter is None: + logger.warning( + "Secondary %s reconnect skipped: adapter unavailable (profile: %s)", + platform.value, profile_name, + ) + return None, None + try: + self._configure_profile_adapter(adapter, profile_name, platform) + success = await self._connect_adapter_with_timeout(adapter, platform, is_reconnect=True) + except BaseException: + # Caller never sees this adapter; release its partial resources here. + await self._safe_adapter_disconnect(adapter, platform) + raise + return adapter, success + + async def _run_secondary_profile_reconnect(self, profile_name: str, platform: Platform) -> None: """Reconnect a retryable secondary adapter under its own profile scope.""" - from gateway.run import _platform_has_bot_credential, _profile_runtime_scope, _reconnect_backoff + from gateway.run import _reconnect_backoff attempts = 0 current_task = asyncio.current_task() try: while self._running: adapter = None try: - from hermes_cli.profiles import get_profile_dir - from hermes_cli.env_loader import hydrate_profile_secret_sources - from gateway.config import load_gateway_config - - profile_home = get_profile_dir(profile_name) - # Like the #16856 MCP discovery path, hydrate external secret - # sources off-loop so they cannot starve platform heartbeats. - await asyncio.to_thread( - hydrate_profile_secret_sources, profile_home - ) - with _profile_runtime_scope(profile_home, hydrate_secrets=False): - profile_config = load_gateway_config().platforms.get(platform) - if profile_config is None or not profile_config.enabled: - return - # Mirrors the startup credential gate: a credential removed from this - # profile's scope must not rebuild an adapter that would fan out turns. - if not _platform_has_bot_credential(platform, profile_config): - logger.info( - "Secondary %s reconnect skipped: no bot credential " - "(profile: %s)", - platform.value, - profile_name, - ) - return - adapter = self._create_adapter(platform, profile_config) - if adapter is None: - logger.warning( - "Secondary %s reconnect skipped: adapter unavailable (profile: %s)", - platform.value, - profile_name, - ) - return - self._configure_profile_adapter( - adapter, profile_name, platform - ) - success = await self._connect_adapter_with_timeout( - adapter, platform, is_reconnect=True - ) - + adapter, success = await self._secondary_reconnect_attempt(profile_name, platform) + if adapter is None: + return if success and self._running: profile_map = self._profile_adapters.setdefault(profile_name, {}) if platform not in profile_map: profile_map[platform] = adapter self._sync_voice_mode_state_to_adapter(adapter) - logger.info( - "✓ %s reconnected (profile: %s)", - platform.value, - profile_name, - ) + logger.info("✓ %s reconnected (profile: %s)", platform.value, profile_name) await self._redeliver_failed_obligations_for_platform( platform, profile=profile_name ) return - # A newer reconnect already won the slot while this - # attempt was awaiting connect; do not replace it. - await self._safe_adapter_disconnect(adapter, platform) - return - - # Shutdown can begin mid-connect(): never republish a newly connected adapter - # after the registry has been drained; release its partial resources instead. - if success: - await self._safe_adapter_disconnect(adapter, platform) - return - + # Not installed: a newer reconnect won the slot mid-connect (never replace it), + # shutdown began mid-connect (never republish after the registry drained), or + # connect failed. Release partial resources; stop only for a non-retryable fatal. await self._safe_adapter_disconnect(adapter, platform) - if ( + if success or ( getattr(adapter, "has_fatal_error", False) and not getattr(adapter, "fatal_error_retryable", True) ): @@ -1472,10 +1165,8 @@ class GatewayAdapterLifecycleMixin: if adapter is not None: await self._safe_adapter_disconnect(adapter, platform) logger.debug( - "Secondary %s reconnect attempt failed (profile: %s)", - platform.value, - profile_name, - exc_info=True, + "Secondary %s reconnect attempt failed (profile: %s)", platform.value, + profile_name, exc_info=True, ) if not self._running: @@ -1483,100 +1174,67 @@ class GatewayAdapterLifecycleMixin: attempts += 1 backoff = _reconnect_backoff(attempts) logger.info( - "Secondary %s reconnect retry in %ds (profile: %s)", - platform.value, - backoff, + "Secondary %s reconnect retry in %ds (profile: %s)", platform.value, backoff, profile_name, ) await asyncio.sleep(backoff) finally: + # Release our slot unless a newer task already owns it. pending = self._profile_failed_platforms - if isinstance(pending, dict): - profile_pending = pending.get(profile_name) - task = profile_pending.get(platform) if isinstance(profile_pending, dict) else None + profile_pending = pending.get(profile_name) if isinstance(pending, dict) else None + if isinstance(profile_pending, dict): + task = profile_pending.get(platform) if not isinstance(task, asyncio.Task) or task is current_task: - if isinstance(profile_pending, dict): - profile_pending.pop(platform, None) - if not profile_pending: - pending.pop(profile_name, None) + profile_pending.pop(platform, None) + if not profile_pending: + pending.pop(profile_name, None) def _schedule_secondary_profile_startup_reconnect( self, profile_name: str, platform: Platform, adapter: BasePlatformAdapter ) -> None: - """Queue a cold-start reconnect for a secondary adapter. - - Startup failures happen BEFORE ``self._running`` flips True, so the regular scheduler's - guard would drop the request. Park a task across startup and hand off to the scheduler once - live (``_profile_failed_platforms`` dedupes); release it if shutdown begins first. - Non-retryable failures are dropped as the regular scheduler would. - """ + """Queue a cold-start reconnect: startup failures happen BEFORE ``_running`` flips True (the + regular scheduler would drop them), so park a task and hand off once live.""" if not getattr(adapter, "fatal_error_retryable", True): return if is_global_startup_conflict(getattr(adapter, "fatal_error_code", None)): - # Same startup contract as the primary path: a live foreign holder of this profile's - # token/identity is an ownership conflict, not a transient blip. Park it fatal (like - # ``duplicate_credential``) instead of retry-storming the token every backoff. + # A live foreign holder of this profile's token is an ownership conflict, not a blip: + # park it fatal (like ``duplicate_credential``) instead of retry-storming. logger.error( "[MULTIPLEX] Profile '%s': %s credential is held by another " - "gateway (%s) — parked, not retried. %s", - profile_name, - platform.value, - adapter.fatal_error_code, - adapter.fatal_error_message or "", + "gateway (%s) — parked, not retried. %s", profile_name, platform.value, + adapter.fatal_error_code, adapter.fatal_error_message or "", ) self._update_platform_runtime_status( - f"{profile_name}:{platform.value}", - platform_state="fatal", - error_code=adapter.fatal_error_code, - error_message=adapter.fatal_error_message, + f"{profile_name}:{platform.value}", platform_state="fatal", + error_code=adapter.fatal_error_code, error_message=adapter.fatal_error_message, ) return + def _handoff() -> None: + try: + self._schedule_secondary_profile_reconnect(profile_name, platform, adapter) + except Exception: + # The handoff touches live registries; if it raises, the parked task dies as an + # unretrieved-task exception logged only at GC. Surface it where operators look. + logger.exception( + "secondary-startup-reconnect handoff failed (profile=%s platform=%s)", + profile_name, platform.value, + ) + async def _await_running_then_schedule() -> None: if self._running: - try: - self._schedule_secondary_profile_reconnect( - profile_name, platform, adapter - ) - except Exception: - # Same GC-time-exception hazard as the post-poll handoff - # below; surface it in gateway.log instead. - logger.exception( - "secondary-startup-reconnect handoff failed " - "(profile=%s platform=%s)", - profile_name, - platform.value, - ) + _handoff() return - # Modest poll: startup completion has no dedicated event, and the reconnect runner's own - # backoff makes sub-100ms precision irrelevant. Bounded so a wedged startup cannot spin. + # Poll (startup completion has no event); bounded so a wedged startup cannot spin. while not self._running and not self._shutdown_event.is_set(): await asyncio.sleep(0.1) if self._running and not self._shutdown_event.is_set(): - try: - self._schedule_secondary_profile_reconnect( - profile_name, platform, adapter - ) - except Exception: - # The handoff touches live registries; if it raises, the parked task dies as an - # unretrieved-task exception logged only at GC. Surface it where operators look. - logger.exception( - "secondary-startup-reconnect handoff failed " - "(profile=%s platform=%s)", - profile_name, - platform.value, - ) + _handoff() - task = asyncio.create_task( + self._retain_background_task(asyncio.create_task( _await_running_then_schedule(), name=f"secondary-startup-reconnect:{profile_name}:{platform.value}", - ) - background_tasks = getattr(self, "_background_tasks", None) - if not isinstance(background_tasks, set): - background_tasks = set() - self._background_tasks = background_tasks - background_tasks.add(task) - task.add_done_callback(background_tasks.discard) + )) def _schedule_secondary_profile_reconnect( self, profile_name: str, platform: Platform, adapter: BasePlatformAdapter @@ -1586,49 +1244,30 @@ class GatewayAdapterLifecycleMixin: return pending = self._profile_failed_platforms if not isinstance(pending, dict): - pending = {} - self._profile_failed_platforms = pending + pending = self._profile_failed_platforms = {} profile_pending = pending.setdefault(profile_name, {}) if platform in profile_pending: return - task = asyncio.create_task( + profile_pending[platform] = self._retain_background_task(asyncio.create_task( self._run_secondary_profile_reconnect(profile_name, platform), name=f"secondary-reconnect:{profile_name}:{platform.value}", - ) - profile_pending[platform] = task - background_tasks = getattr(self, "_background_tasks", None) - if not isinstance(background_tasks, set): - background_tasks = set() - self._background_tasks = background_tasks - background_tasks.add(task) - task.add_done_callback(background_tasks.discard) + )) def _make_profile_fatal_error_handler( self, profile_name: str, platform: Platform ) -> Callable[[BasePlatformAdapter], Awaitable[None]]: """Route a secondary-profile fatal error to that profile's reconnect slot.""" - async def _handler(adapter: BasePlatformAdapter) -> None: - await self._handle_profile_adapter_fatal_error(profile_name, platform, adapter) - - return _handler + return functools.partial(self._handle_profile_adapter_fatal_error, profile_name, platform) async def _handle_profile_adapter_fatal_error( - self, - profile_name: str, - platform: Platform, - adapter: BasePlatformAdapter, + self, profile_name: str, platform: Platform, adapter: BasePlatformAdapter ) -> None: - """Remove a failed multiplexed adapter without touching the primary slot. - - Secondaries live in ``_profile_adapters``, which the primary-only fatal handler ignores; - without this route a fatal secondary Discord client stayed live forever. - """ + """Remove a failed multiplexed adapter (the primary-only fatal handler ignores them).""" profile_map = getattr(self, "_profile_adapters", {}).get(profile_name) if not isinstance(profile_map, dict) or profile_map.get(platform) is not adapter: logger.debug( "Ignoring stale fatal error from secondary %s adapter (profile: %s)", - platform.value, - profile_name, + platform.value, profile_name, ) return profile_map.pop(platform, None) @@ -1637,94 +1276,87 @@ class GatewayAdapterLifecycleMixin: return self._schedule_secondary_profile_reconnect(profile_name, platform, adapter) logger.error( - "Fatal %s adapter error for multiplexed profile %s (%s)", - platform.value, - profile_name, + "Fatal %s adapter error for multiplexed profile %s (%s)", platform.value, profile_name, adapter.fatal_error_code or "unknown", ) - def _make_profile_message_handler(self, profile_name: str): - """Return a message handler that stamps source.profile then delegates. - - Auth runs inside ``_handle_message`` *before* the agent-turn scope is installed. For - secondary profiles under multiplex, wrap the whole handler in ``_profile_runtime_scope`` - so allowlists/tokens from that profile's ``.env`` are visible to ``get_secret`` / authz. - """ - from gateway.run import _async_profile_runtime_scope + @staticmethod + def _profile_home_or_none(profile_name: str): from hermes_cli.profiles import get_profile_dir - try: - profile_home = get_profile_dir(profile_name) + return get_profile_dir(profile_name) except Exception: - profile_home = None + return None + + @staticmethod + def _stamp_event_profile(event, profile_name: str) -> None: + """Best-effort: stamp ``source.profile`` on an inbound event that has none yet.""" + try: + if getattr(event, "source", None) is not None and not event.source.profile: + event.source.profile = profile_name + except Exception: + pass + + def _make_profile_message_handler(self, profile_name: str): + """Message handler that stamps source.profile, then delegates under the profile scope + (auth runs BEFORE the agent-turn scope, so the profile's ``.env`` must be visible here).""" + from gateway.run import _async_profile_runtime_scope + profile_home = self._profile_home_or_none(profile_name) async def _handler(event): - try: - if getattr(event, "source", None) is not None and not event.source.profile: - event.source.profile = profile_name - except Exception: - pass - if profile_home is not None: - async with _async_profile_runtime_scope(profile_home): - return await self._handle_message(event) - return await self._handle_message(event) + self._stamp_event_profile(event, profile_name) + async with ( + _async_profile_runtime_scope(profile_home) if profile_home is not None + else contextlib.nullcontext() + ): + return await self._handle_message(event) return _handler def _make_profile_busy_session_handler(self, profile_name: str): """Stamp an owning adapter's profile before resolving busy policy.""" async def _handler(event, _session_key): - try: - if getattr(event, "source", None) is not None and not event.source.profile: - event.source.profile = profile_name - except Exception: - pass + self._stamp_event_profile(event, profile_name) routed_session_key = self._session_key_for_source(event.source) - return await self._handle_active_session_busy_message( - event, routed_session_key - ) + return await self._handle_active_session_busy_message(event, routed_session_key) return _handler def _make_default_profile_message_handler(self): - """Scope primary-adapter messages to their routed multiplex profile. - - Resolve the home per event so session lookup and transcript loading use the same profile - store as the agent run. Authorization stays with the transport profile (a routed profile - may intentionally have no bot credential/allowlist): the transport home is preserved on the - live source and never re-checked against the routed scope. Unrouted events keep the default. - """ + """Scope primary-adapter messages to their routed multiplex profile. Authorization stays + with the transport profile (a routed profile may have no credential/allowlist).""" from gateway.run import _async_profile_runtime_scope, get_hermes_home default_home = Path(get_hermes_home()) async def _handler(event): source = event.source - # In-process only (SessionSource serialization ignores dynamic attrs). The route selects - # agent/session state, not which bot admitted the message — separate trust domains. + # In-process only (serialization ignores dynamic attrs); route ≠ admitting bot. source._authorization_profile_home = default_home if ( not getattr(source, "profile", None) and getattr(source, "profile_route_rejected", False) is not True + and not self._stamp_routed_profile(source) ): - from gateway.profile_routing import ProfileRouteRejected - - try: - source.profile = self._profile_name_for_source(source) - except ProfileRouteRejected: - # NOT write-only: the ``_handle_message`` ingress gate reads this exact marker - # and drops the message fail-closed (explicit route to an unserved profile). - source.profile_route_rejected = True - + # Read by the ``_handle_message`` ingress gate, which drops fail-closed. + source.profile_route_rejected = True profile_home = ( self._resolve_profile_home_for_source(source) - if getattr(source, "profile", None) - else default_home + if getattr(source, "profile", None) else default_home ) async with _async_profile_runtime_scope(profile_home): return await self._handle_message(event) return _handler + def _stamp_routed_profile(self, source) -> bool: + """Stamp ``source.profile`` from ``profile_routes``; False when the route is rejected.""" + from gateway.profile_routing import ProfileRouteRejected + try: + source.profile = self._profile_name_for_source(source) + except ProfileRouteRejected: + return False + return True + def _primary_message_handler(self): """Return the correctly scoped handler for a primary adapter.""" if getattr(self.config, "multiplex_profiles", False): @@ -1735,12 +1367,8 @@ class GatewayAdapterLifecycleMixin: """Authorize and publish one normalized adapter event to plugin hooks.""" try: from hermes_cli.lifecycle import has_hook, invoke_hook - - if not has_hook("gateway_platform_event"): - return - if not self._is_user_authorized_for_source(source): - return - invoke_hook("gateway_platform_event", **event) + if has_hook("gateway_platform_event") and self._is_user_authorized_for_source(source): + invoke_hook("gateway_platform_event", **event) except Exception: # Observer failures must never break the adapter's update loop. logger.debug("gateway_platform_event hook dispatch failed", exc_info=True) @@ -1748,20 +1376,16 @@ class GatewayAdapterLifecycleMixin: def _make_profile_platform_event_handler(self, profile_name: str): """Bind platform-event auth and hook dispatch to one multiplex profile.""" from gateway.run import _profile_runtime_scope - from hermes_cli.profiles import get_profile_dir - - try: - profile_home = get_profile_dir(profile_name) - except Exception: - profile_home = None + profile_home = self._profile_home_or_none(profile_name) async def _handler(event, source): if getattr(source, "profile", None) is None: source.profile = profile_name - if profile_home is not None: - with _profile_runtime_scope(profile_home): - return await self._handle_gateway_platform_event(event, source) - return await self._handle_gateway_platform_event(event, source) + with ( + _profile_runtime_scope(profile_home) if profile_home is not None + else contextlib.nullcontext() + ): + return await self._handle_gateway_platform_event(event, source) return _handler @@ -1783,188 +1407,103 @@ class GatewayAdapterLifecycleMixin: return self._handle_gateway_platform_event @staticmethod - def _adapter_credential_claim( - platform: Platform, adapter: Any - ) -> Optional[tuple]: + def _adapter_credential_claim(platform: Platform, adapter: Any) -> Optional[tuple]: """Return the exclusive credential resource claimed by an adapter.""" from gateway.run import GatewayRunner fingerprint = GatewayRunner._adapter_credential_fingerprint(adapter) - if fingerprint is None: - return None - return (platform, fingerprint) + return None if fingerprint is None else (platform, fingerprint) @staticmethod def _adapter_listener_claim(platform: Platform, adapter: Any) -> Optional[tuple]: - """Return the exclusive listener resource claimed by an adapter. - - Sidecars with different credentials still cannot share a bind+port; expose it as a claim so - multiplex startup rejects the later adapter before connect()/disconnect() disturb the first. - """ - if getattr(platform, "value", None) != "photon": - return None + """Exclusive listener claim (Photon sidecar bind+port): distinct credentials still cannot + share a port, so the later adapter is rejected before connect() disturbs the first.""" bind = getattr(adapter, "_sidecar_bind", None) - port = getattr(adapter, "_sidecar_port", None) - if not isinstance(bind, str) or not bind.strip(): + if getattr(platform, "value", None) != "photon" or not isinstance(bind, str) or not bind.strip(): return None try: - port = int(port) + port = int(getattr(adapter, "_sidecar_port", None)) except (TypeError, ValueError): return None return ("listener", "photon", bind.strip().lower(), port) @staticmethod def _adapter_credential_fingerprint(adapter: Any) -> Optional[str]: - """Return a stable, log-safe fingerprint of an adapter's credential. - - Salted hash (never the credential) used to detect two profiles sharing one platform - credential; None when no credential is discoverable (conflict detection is then skipped). - """ - token = None - for attr in ( - "token", - "bot_token", - "_token", - "api_token", - "_bot_token", - # Photon/Spectrum authenticates with project credentials, not a bot token; including - # its secret stops multiplexed profiles spawning rival sidecars for one account/port. - "_project_secret", - # Feishu/Lark authenticates with an app_id/app_secret pair (one WebSocket per app). - # app_id is stable, log-safe and already the adapter's _app_lock_identity, so including - # it lets the multiplex guard refuse cloned profiles competing for the same app. - "_app_id", - # Same class: Teams (client_id/client_secret) and WeCom - # (bot_id/secret) authenticate with an app-style id pair too. - "_client_id", - "_bot_id", - ): - val = getattr(adapter, attr, None) + """Salted, log-safe hash of an adapter's credential; None when none is discoverable + (conflict detection is then skipped).""" + # Many adapters (e.g. Discord) keep the token on `config`; without that fallback the + # same-token check is silently skipped and every profile polls the same bot. + candidates = [ + (adapter, attr) for attr in ( + "token", "bot_token", "_token", "api_token", "_bot_token", + "_project_secret", # Photon/Spectrum: project credentials, not a bot token + "_app_id", # Feishu/Lark app_id — stable, log-safe, already the _app_lock_identity + "_client_id", "_bot_id", # Teams / WeCom app-style id pairs + ) + ] + [(getattr(adapter, "config", None), attr) for attr in ("token", "bot_token")] + for obj, attr in candidates: + val = getattr(obj, attr, None) if isinstance(val, str) and val.strip(): - token = val.strip() - break - # Many adapters (e.g. Discord) store the token on their `config` sub-object. Without this - # lookup they return None, the same-token check is silently skipped, and every profile's - # adapter polls the same bot token — a per-message race over which one answers. - if not token: - cfg = getattr(adapter, "config", None) - if cfg is not None: - for attr in ("token", "bot_token"): - val = getattr(cfg, attr, None) - if isinstance(val, str) and val.strip(): - token = val.strip() - break - if not token: - config = getattr(adapter, "config", None) - val = getattr(config, "token", None) - if isinstance(val, str) and val.strip(): - token = val.strip() - if not token: - return None - import hashlib - return hashlib.sha256(("hermes-mux:" + token).encode("utf-8")).hexdigest()[:16] + import hashlib + return hashlib.sha256(("hermes-mux:" + val.strip()).encode("utf-8")).hexdigest()[:16] + return None - def _create_adapter( - self, - platform: Platform, - config: Any, - ) -> Optional[BasePlatformAdapter]: - """Create an adapter and bind it to this gateway runner. - - Every lifecycle path (primary/secondary startup, reconnect) uses this method; keep runner - binding here so adapters can resolve inbound profile routes before handlers or connect(). - """ + def _create_adapter(self, platform: Platform, config: Any) -> Optional[BasePlatformAdapter]: + """Create an adapter bound to this runner (every lifecycle path goes through here so + adapters can resolve inbound profile routes before handlers or connect()).""" adapter = self._instantiate_adapter(platform, config) if adapter is not None: adapter.gateway_runner = self return adapter - def _instantiate_adapter( - self, - platform: Platform, - config: Any, - ) -> Optional[BasePlatformAdapter]: - """Instantiate the appropriate adapter for a platform. - - Checks platform_registry (plugin adapters) first, then the built-in table of core platforms. - """ + def _instantiate_adapter(self, platform: Platform, config: Any) -> Optional[BasePlatformAdapter]: + """Instantiate the adapter for a platform: plugin registry first, then built-ins.""" from gateway.run import _instantiate_builtin_adapter if hasattr(config, "extra") and isinstance(config.extra, dict): + config.extra.setdefault("group_sessions_per_user", self.config.group_sessions_per_user) config.extra.setdefault( - "group_sessions_per_user", - self.config.group_sessions_per_user, - ) - config.extra.setdefault( - "thread_sessions_per_user", - getattr(self.config, "thread_sessions_per_user", False), + "thread_sessions_per_user", getattr(self.config, "thread_sessions_per_user", False) ) - # ── Plugin-registered platforms (checked first) ─────────────────── try: from gateway.platform_registry import platform_registry if platform_registry.is_registered(platform.value): adapter = platform_registry.create_adapter(platform.value, config) - if adapter is not None: - return adapter - # Registered but failed to instantiate — don't silently fall - # through to built-ins (there are none for plugin platforms). - logger.error( - "Platform '%s' is registered but adapter creation failed " - "(check dependencies and config)", - platform.value, - ) - return None + if adapter is None: # registered but failed — never fall through to built-ins + logger.error( + "Platform '%s' is registered but adapter creation failed " + "(check dependencies and config)", platform.value, + ) + return adapter except Exception as e: logger.debug("Platform registry lookup for '%s' failed: %s", platform.value, e) - # Fall through to built-in adapters below - return _instantiate_builtin_adapter(platform, config) def _make_adapter_auth_check( - self, - platform: Platform, - profile_name: Optional[str] = None, + self, platform: Platform, profile_name: Optional[str] = None ) -> Callable[[str, Optional[str], Optional[str]], bool]: - """Build a platform-bound auth callback for adapter use. - - Adapters fetching external context (e.g. Slack ``conversations.replies``) use it via - ``_is_sender_authorized`` to mark non-allowlisted senders unverified (prompt-injection - mitigation). Delegates to :meth:`_is_user_authorized` so the full auth chain stays the single - source of truth. ``profile_name`` binds a secondary adapter to its own secret scope; for the - shared primary (None) the ``profile_routes`` match is stamped on the source so the routed - profile's pairing store is consulted while allowlist reads stay under the transport home. - """ + """Platform-bound auth callback for adapters (prompt-injection mitigation for fetched + context); delegates to :meth:`_is_user_authorized`. ``profile_name`` binds a secondary to + its scope; for the shared primary (None) the routed profile is stamped so its pairing store + is consulted while allowlist reads stay under the transport home.""" from gateway.run import get_hermes_home multiplex = bool(getattr(self.config, "multiplex_profiles", False)) - transport_home = ( - Path(get_hermes_home()) if multiplex and profile_name is None else None - ) + transport_home = (Path(get_hermes_home()) if multiplex and profile_name is None else None) def check( - user_id: str, - chat_type: Optional[str] = None, - chat_id: Optional[str] = None, - *, - is_bot: bool = False, - thread_id: Optional[str] = None, + user_id: str, chat_type: Optional[str] = None, chat_id: Optional[str] = None, *, + is_bot: bool = False, thread_id: Optional[str] = None, ) -> bool: if not user_id: return False source = SessionSource( - platform=platform, - chat_id=chat_id or "", - chat_type=chat_type or "group", - user_id=user_id, - thread_id=thread_id, - is_bot=bool(is_bot), - profile=profile_name, + platform=platform, chat_id=chat_id or "", chat_type=chat_type or "group", + user_id=user_id, thread_id=thread_id, is_bot=bool(is_bot), profile=profile_name, ) - # Same in-process transport provenance ``build_source`` retains, so adapter-level policy - # reads (config.yaml group_allowed_chats, allow_from) resolve the receiving adapter even - # once the routed profile is stamped below. + # Same transport provenance ``build_source`` retains, so adapter-level policy reads + # resolve the receiving adapter even once the routed profile is stamped below. registry = ( (getattr(self, "_profile_adapters", None) or {}).get(profile_name) - if profile_name - else getattr(self, "adapters", None) + if profile_name else getattr(self, "adapters", None) ) or {} adapter = registry.get(platform) if adapter is not None: @@ -1972,13 +1511,7 @@ class GatewayAdapterLifecycleMixin: if transport_home is None: return self._is_user_authorized(source) source._authorization_profile_home = transport_home - from gateway.profile_routing import ProfileRouteRejected - - try: - source.profile = self._profile_name_for_source(source) - except ProfileRouteRejected: - # Same fail-closed outcome as the ingress gate in - # ``_handle_message`` for a route to an unserved profile. - return False + if not self._stamp_routed_profile(source): + return False # fail-closed, like the ``_handle_message`` ingress gate return self._is_user_authorized_for_source(source) return check diff --git a/gateway/run_agent_cache.py b/gateway/run_agent_cache.py index d8be1190dc..4aabc3ba7a 100644 --- a/gateway/run_agent_cache.py +++ b/gateway/run_agent_cache.py @@ -1,4 +1,4 @@ -"""Agent cache, session model overrides, turn leases, run generations and conversation-scope reset methods for GatewayRunner. +"""Agent cache, session model overrides, turn leases, run generations and conversation-scope reset for GatewayRunner. Split out of ``gateway/run.py``; bound onto ``GatewayRunner`` via the MRO. ``gateway.run`` internals are imported lazily inside method bodies (import cycle), @@ -7,16 +7,17 @@ so ``patch("gateway.run.X")`` keeps intercepting them at call time. from __future__ import annotations -import logging -from typing import TYPE_CHECKING +import importlib import inspect +import logging import threading import time -from contextlib import suppress +from contextlib import nullcontext, suppress +from typing import TYPE_CHECKING, Any, Dict, List, Optional + from gateway.config import Platform from gateway.session import SessionSource, build_session_context_prompt from hermes_cli.config import cfg_get -from typing import Any, Dict, List, Optional if TYPE_CHECKING: # string annotations only; never imported at runtime (cycle) from gateway.run import GatewayRunner, TurnRunner # noqa: F401 @@ -24,20 +25,25 @@ if TYPE_CHECKING: # string annotations only; never imported at runtime (cycle) # Log-record parity with the origin module. logger = logging.getLogger("gateway.run") +# Override fields layered onto runtime kwargs when non-None (partial overrides don't clobber defaults). +_OVERRIDE_APPLY_KEYS = ( + "provider", "requested_provider", "api_key", "base_url", "api_mode", "credential_pool", "capabilities", "max_tokens", +) + + +def _first_agent(entry: Any) -> Any: + """Unwrap a cache entry (``(agent, sig, ...)`` tuple or bare agent) to its agent.""" + return entry[0] if isinstance(entry, tuple) and entry else entry + class GatewayAgentCacheMixin: - """Agent cache, session model overrides, turn leases, run generations and conversation-scope reset methods for GatewayRunner.""" - - @classmethod - def _empty_honcho_cache_busting_config(cls) -> dict[str, Any]: - return {key: None for key in cls._HONCHO_CACHE_BUSTING_KEYS} + """Agent cache, session model overrides, turn leases, run generations and conversation-scope reset for GatewayRunner.""" @classmethod def _extract_honcho_cache_busting_config(cls) -> dict[str, Any]: - """Extract Honcho identity keys, memoized by honcho.json mtime.""" + """Extract Honcho identity keys, memoized by honcho.json mtime; all-None when unavailable.""" try: from plugins.memory.honcho.client import HonchoClientConfig, resolve_config_path - path = resolve_config_path() try: mtime_ns = path.stat().st_mtime_ns @@ -60,7 +66,7 @@ class GatewayAgentCacheMixin: cls._HONCHO_CACHE_BUSTING_MEMO = {memo_key: values} return dict(values) except Exception: - return cls._empty_honcho_cache_busting_config() + return dict.fromkeys(cls._HONCHO_CACHE_BUSTING_KEYS) @classmethod def _extract_cache_busting_config(cls, user_config: dict | None) -> dict: @@ -75,8 +81,7 @@ class GatewayAgentCacheMixin: for section, key in cls._CACHE_BUSTING_CONFIG_KEYS: section_val = cfg.get(section) if section == "checkpoints" and isinstance(section_val, bool): - # Preserve legacy ``checkpoints: true`` behavior. A live - # toggle must still rebuild the cached agent. + # Legacy ``checkpoints: true``: a live toggle must still rebuild the cached agent. out[f"{section}.{key}"] = section_val if key == "enabled" else None elif isinstance(section_val, dict): out[f"{section}.{key}"] = section_val.get(key) @@ -84,30 +89,24 @@ class GatewayAgentCacheMixin: out[f"{section}.{key}"] = None try: from tools.registry import registry - out["tools.registry_generation"] = getattr(registry, "_generation", None) except Exception: out["tools.registry_generation"] = None - # Honcho identity-mapping keys live in honcho.json, not user_config. - # Only read that file when Honcho is the active memory provider. + # Honcho identity-mapping keys live in honcho.json, not user_config; only read that file + # when Honcho is the active memory provider. provider = cfg_get(cfg, "memory", "provider") if isinstance(provider, str) and provider.lower() == "honcho": out.update(cls._extract_honcho_cache_busting_config()) else: - out.update(cls._empty_honcho_cache_busting_config()) + out.update(dict.fromkeys(cls._HONCHO_CACHE_BUSTING_KEYS)) return out @staticmethod def _agent_config_signature( - model: str, - runtime: dict, - enabled_toolsets: list, - ephemeral_prompt: str, - cache_keys: dict | None = None, - user_id: str | None = None, - user_id_alt: str | None = None, + model: str, runtime: dict, enabled_toolsets: list, ephemeral_prompt: str, + cache_keys: dict | None = None, user_id: str | None = None, user_id_alt: str | None = None, skip_context_files: bool = False, ) -> str: """Compute a stable string key from agent config values. @@ -119,14 +118,11 @@ class GatewayAgentCacheMixin: (``thread_sessions_per_user=False``) would attribute one user's messages to another's peer. """ import hashlib, json as _j - # Fingerprint the FULL credential, not a short prefix: OAuth/JWT-style tokens often share a # common prefix (e.g. "eyJhbGci"), so a prefix would give false cache hits across auth switches. _api_key = str(runtime.get("api_key", "") or "") _api_key_fingerprint = hashlib.sha256(_api_key.encode()).hexdigest() if _api_key else "" - _cache_keys_sorted = sorted((cache_keys or {}).items()) - blob = _j.dumps( [ model, @@ -137,10 +133,9 @@ class GatewayAgentCacheMixin: runtime.get("api_mode", ""), sorted((runtime.get("capabilities") or {}).items()), sorted(enabled_toolsets) if enabled_toolsets else [], - # reasoning_config excluded — it's set per-message on the - # cached agent and doesn't affect system prompt or tools. + # reasoning_config excluded — set per-message on the cached agent; no prompt/tool effect. ephemeral_prompt or "", - _cache_keys_sorted, + sorted((cache_keys or {}).items()), str(user_id or ""), str(user_id_alt or ""), # skip_context_files changes the agent's frozen system prompt (context files in vs out): @@ -152,6 +147,11 @@ class GatewayAgentCacheMixin: ) return hashlib.sha256(blob.encode()).hexdigest()[:16] + def _session_model_override(self, session_key: str) -> Optional[dict]: + """Current in-memory /model override for ``session_key`` (None when absent).""" + state = self._peek_session_state(session_key) + return state.conversation.model_override if state else None + def _rehydrate_session_model_override(self, session_key: str) -> None: """Lazily restore a persisted /model override after a gateway restart. @@ -160,11 +160,7 @@ class GatewayAgentCacheMixin: is never persisted and is re-resolved. No-op when an in-memory override or nothing exists. """ from gateway.run import _resolve_runtime_agent_kwargs_for_provider - _rehydrate_state = self._peek_session_state(session_key) - if ( - _rehydrate_state is not None - and _rehydrate_state.conversation.model_override is not None - ): + if self._session_model_override(session_key) is not None: return store = getattr(self, "session_store", None) if store is None: @@ -172,30 +168,21 @@ class GatewayAgentCacheMixin: try: persisted = store.get_model_override(session_key) except Exception: - logger.debug( - "Failed to read persisted session model override", exc_info=True - ) + logger.debug("Failed to read persisted session model override", exc_info=True) return if not persisted: return - override: Dict[str, Any] = { - "model": persisted.get("model"), - "provider": persisted.get("provider"), - "base_url": persisted.get("base_url"), - } + override: Dict[str, Any] = {k: persisted.get(k) for k in ("model", "provider", "base_url")} provider = persisted.get("provider") if provider: - # Re-resolve credentials for the persisted provider. On failure (e.g. credentials - # removed since the switch) keep the credential-less override — - # _resolve_session_agent_runtime falls back to env resolution and layers model/provider. + # Re-resolve credentials for the persisted provider. On failure (e.g. credentials removed + # since the switch) keep the credential-less override — _resolve_session_agent_runtime + # falls back to env resolution and layers model/provider. try: runtime = _resolve_runtime_agent_kwargs_for_provider(provider) - override["api_key"] = runtime.get("api_key") - override["api_mode"] = runtime.get("api_mode") - override["credential_pool"] = runtime.get("credential_pool") - override["request_overrides"] = dict( - runtime.get("request_overrides") or {} - ) + for k in ("api_key", "api_mode", "credential_pool"): + override[k] = runtime.get(k) + override["request_overrides"] = dict(runtime.get("request_overrides") or {}) override["requested_provider"] = runtime.get("requested_provider") override["capabilities"] = dict(runtime.get("capabilities") or {}) override["max_tokens"] = runtime.get("max_tokens") @@ -204,8 +191,7 @@ class GatewayAgentCacheMixin: except Exception: logger.debug( "Credential re-resolution failed for persisted override " - "(provider=%s); using credential-less override", - provider, exc_info=True, + "(provider=%s); using credential-less override", provider, exc_info=True, ) self._session_state(session_key).conversation.model_override = override logger.info( @@ -213,30 +199,18 @@ class GatewayAgentCacheMixin: session_key, override.get("model"), provider or "", ) - def _apply_session_model_override( - self, session_key: str, model: str, runtime_kwargs: dict - ) -> tuple: + def _apply_session_model_override(self, session_key: str, model: str, runtime_kwargs: dict) -> tuple: """Apply /model session overrides if present, returning (model, runtime_kwargs). Overrides take precedence over config.yaml defaults so the switched model is actually used; ``None`` fields are skipped so partial overrides don't clobber valid defaults. """ from gateway.run import _credential_pool_for_provider - _apply_state = self._peek_session_state(session_key) - override = _apply_state.conversation.model_override if _apply_state else None + override = self._session_model_override(session_key) if not override: return model, runtime_kwargs model = override.get("model", model) - for key in ( - "provider", - "requested_provider", - "api_key", - "base_url", - "api_mode", - "credential_pool", - "capabilities", - "max_tokens", - ): + for key in _OVERRIDE_APPLY_KEYS: val = override.get(key) if val is not None: runtime_kwargs[key] = val @@ -244,25 +218,19 @@ class GatewayAgentCacheMixin: # it (even as None) so switching to a provider without configured overrides clears a stale # value left by the default provider's runtime resolution. if "request_overrides" in override: - override_request_overrides = override.get("request_overrides") - if isinstance(override_request_overrides, dict) and override_request_overrides: - runtime_kwargs["request_overrides"] = dict(override_request_overrides) - else: - runtime_kwargs["request_overrides"] = override_request_overrides + ro = override.get("request_overrides") + runtime_kwargs["request_overrides"] = dict(ro) if isinstance(ro, dict) and ro else ro if ( runtime_kwargs.get("api_key") and runtime_kwargs.get("credential_pool") is None and override.get("provider") ): - runtime_kwargs["credential_pool"] = _credential_pool_for_provider( - override.get("provider") - ) + runtime_kwargs["credential_pool"] = _credential_pool_for_provider(override.get("provider")) return model, runtime_kwargs def _snapshot_session_model_override(self, session_key: str) -> dict: """Capture a gateway session override before a one-turn switch.""" - _snap_state = self._peek_session_state(session_key) - override = _snap_state.conversation.model_override if _snap_state else None + override = self._session_model_override(session_key) return { "had_override": override is not None, "override": dict(override) if override is not None else None, @@ -273,9 +241,7 @@ class GatewayAgentCacheMixin: if not session_key: return if snapshot.get("had_override"): - self._session_state(session_key).conversation.model_override = dict( - snapshot.get("override") or {} - ) + self._session_state(session_key).conversation.model_override = dict(snapshot.get("override") or {}) else: _rst_state = self._peek_session_state(session_key) if _rst_state is not None: @@ -284,15 +250,11 @@ class GatewayAgentCacheMixin: def _is_intentional_model_switch(self, session_key: str, agent_model: str) -> bool: """Return True if *agent_model* matches an active /model session override.""" - _ims_state = self._peek_session_state(session_key) - override = _ims_state.conversation.model_override if _ims_state else None + override = self._session_model_override(session_key) return override is not None and override.get("model") == agent_model def _release_running_agent_state( - self, - session_key: str, - *, - run_generation: Optional[int] = None, + self, session_key: str, *, run_generation: Optional[int] = None ) -> bool: """Pop ALL per-running-agent state entries for ``session_key``; True when cleared. @@ -303,9 +265,7 @@ class GatewayAgentCacheMixin: """ if not session_key: return False - if run_generation is not None and not self._is_session_run_current( - session_key, run_generation - ): + if run_generation is not None and not self._is_session_run_current(session_key, run_generation): return False state = self._peek_session_state(session_key) if state is not None: @@ -314,9 +274,7 @@ class GatewayAgentCacheMixin: try: lease.release() except Exception: - logger.debug( - "Failed to release active session slot", exc_info=True - ) + logger.debug("Failed to release active session slot", exc_info=True) # One structured reset instead of a drifting pop-list. Turn-lease tokens are deliberately NOT # cleared here — _release_turn_lease owns them. state.turn.clear() @@ -325,21 +283,29 @@ class GatewayAgentCacheMixin: self._persist_active_agents() return True + def _held_turn_lease(self, session_key: str, run_generation: int): + """Return ``(registry, turn)`` when ``session_key`` holds a lease token for ``run_generation``, else None.""" + if not session_key: + return None + registry = getattr(self, "_turn_leases", None) + state = self._peek_session_state(session_key) + if state is None or registry is None: + return None + turn = state.turn + if turn.lease_token is None or turn.lease_generation != run_generation: + return None + return registry, turn + def _release_turn_lease(self, session_key: str, run_generation: int) -> bool: """Release the turn lease acquired by (``session_key``, ``run_generation``). Token map is keyed by (routing key, run generation), so a stale unwind pops only ITS token and the registry's identity check refuses it if a newer turn holds the lease. Idempotent. """ - if not session_key: - return False - registry = getattr(self, "_turn_leases", None) - state = self._peek_session_state(session_key) - if state is None or registry is None: - return False - turn = state.turn - if turn.lease_token is None or turn.lease_generation != run_generation: + held = self._held_turn_lease(session_key, run_generation) + if held is None: return False + registry, turn = held token = turn.lease_token turn.lease_token = None turn.lease_generation = None @@ -349,24 +315,17 @@ class GatewayAgentCacheMixin: logger.debug("Failed to release turn lease", exc_info=True) return False - def _rebind_turn_lease( - self, session_key: str, run_generation: int, new_session_id: str - ) -> bool: + def _rebind_turn_lease(self, session_key: str, run_generation: int, new_session_id: str) -> bool: """Follow a mid-turn session_id rotation with the held turn lease. Compression can rotate ``session_entry.session_id`` mid-turn; the flush targets the NEW id, so the serialization boundary must follow or an alias key resolving the new id could start a concurrent turn the lease never sees. Call at every mid-turn reassignment; no-op if no token. """ - if not session_key or not new_session_id: - return False - registry = getattr(self, "_turn_leases", None) - state = self._peek_session_state(session_key) - if state is None or registry is None: - return False - turn = state.turn - if turn.lease_token is None or turn.lease_generation != run_generation: + held = self._held_turn_lease(session_key, run_generation) if new_session_id else None + if held is None: return False + registry, turn = held try: return registry.rebind(turn.lease_token, new_session_id) except Exception: @@ -386,8 +345,6 @@ class GatewayAgentCacheMixin: from gateway.run import _CONVERSATION_SCOPED_STATE if not session_key: return - # Structural clear: every conversation-scoped field resets in one - # call — no per-attribute pop-list to drift. state = self._peek_session_state(session_key) if state is not None: state.conversation.clear() @@ -399,18 +356,14 @@ class GatewayAgentCacheMixin: if isinstance(store, dict): store.pop(session_key, None) self._clear_session_boundary_security_state(session_key) - logger.debug( - "Cleared conversation scope for %s (%s)", session_key, reason - ) + logger.debug("Cleared conversation scope for %s (%s)", session_key, reason) def _clear_session_boundary_security_state(self, session_key: str) -> None: """Clear per-session control state that must not survive a boundary switch.""" if not session_key: return - pending_skills_reload_notes = getattr( - self, "_pending_skills_reload_notes", None - ) + pending_skills_reload_notes = getattr(self, "_pending_skills_reload_notes", None) if isinstance(pending_skills_reload_notes, dict): pending_skills_reload_notes.pop(session_key, None) @@ -419,33 +372,17 @@ class GatewayAgentCacheMixin: _sec_state.persistent.approvals = None _sec_state.persistent.update_prompt_pending = False - try: - from tools import slash_confirm as _slash_confirm_mod - except Exception: - _slash_confirm_mod = None - if _slash_confirm_mod is not None: + for mod, attr, what in ( + ("tools.slash_confirm", "clear", "slash-confirm"), ("tools.approval", "clear_session", "approval"), + ): try: - _slash_confirm_mod.clear(session_key) + clear = getattr(importlib.import_module(mod), attr) + except Exception: + continue + try: + clear(session_key) except Exception as e: - logger.debug( - "Failed to clear slash-confirm state for session boundary %s: %s", - session_key, - e, - ) - - try: - from tools.approval import clear_session as _clear_approval_session - except Exception: - return - - try: - _clear_approval_session(session_key) - except Exception as e: - logger.debug( - "Failed to clear approval state for session boundary %s: %s", - session_key, - e, - ) + logger.debug("Failed to clear %s state for session boundary %s: %s", what, session_key, e) def _begin_session_run_generation(self, session_key: str) -> int: """Claim a fresh, monotonically increasing run generation token for ``session_key``. @@ -464,12 +401,7 @@ class GatewayAgentCacheMixin: """Invalidate any in-flight run token for ``session_key``.""" generation = self._begin_session_run_generation(session_key) if reason: - logger.info( - "Invalidated run generation for %s → %d (%s)", - session_key, - generation, - reason, - ) + logger.info("Invalidated run generation for %s → %d (%s)", session_key, generation, reason) return generation def _is_session_run_current(self, session_key: str, generation: int) -> bool: @@ -480,12 +412,7 @@ class GatewayAgentCacheMixin: current = state.persistent.run_generation if state is not None else 0 return int(current) == int(generation) - def _bind_adapter_run_generation( - self, - adapter: Any, - session_key: str, - generation: int | None, - ) -> None: + def _bind_adapter_run_generation(self, adapter: Any, session_key: str, generation: int | None) -> None: """Bind a gateway run generation to the adapter's active-session event.""" if not adapter or not session_key or generation is None: return @@ -497,13 +424,8 @@ class GatewayAgentCacheMixin: pass async def _interrupt_and_clear_session( - self, - session_key: str, - source: SessionSource, - *, - interrupt_reason: str, - invalidation_reason: str, - release_running_state: bool = True, + self, session_key: str, source: SessionSource, *, interrupt_reason: str, + invalidation_reason: str, release_running_state: bool = True, ) -> None: """Interrupt the current run and clear queued session state consistently.""" from gateway.run import _AGENT_PENDING_SENTINEL, _reap_gateway_turn_processes, request_hard_interrupt @@ -515,50 +437,37 @@ class GatewayAgentCacheMixin: _process_baseline = None if running_agent and running_agent is not _AGENT_PENDING_SENTINEL: request_hard_interrupt(running_agent, interrupt_reason) - _process_task_id = getattr( - running_agent, "_gateway_turn_process_task_id", "" - ) - _process_baseline = getattr( - running_agent, "_gateway_turn_process_baseline", None - ) + _process_task_id = getattr(running_agent, "_gateway_turn_process_task_id", "") + _process_baseline = getattr(running_agent, "_gateway_turn_process_baseline", None) # Bump the generation BEFORE scheduling the reap thread and capture the post-bump value: # task_id is session-scoped, so a replacement turn spawning before the reap runs bumps it # again and the closure sees a stale generation and skips — the replacement's own baseline # covers its cleanup, so nothing stays unreaped. - _generation_at_interrupt = self._invalidate_session_run_generation( - session_key, reason=invalidation_reason - ) + _generation_at_interrupt = self._invalidate_session_run_generation(session_key, reason=invalidation_reason) if _process_task_id and _process_baseline is not None: threading.Thread( target=_reap_gateway_turn_processes, args=(_process_task_id, _process_baseline), kwargs={ "source": "gateway_turn_interrupt", - "is_still_current": lambda: self._is_session_run_current( - session_key, _generation_at_interrupt - ), + "is_still_current": lambda: self._is_session_run_current(session_key, _generation_at_interrupt), }, name=f"gateway-turn-reaper-{_process_task_id[:12]}", daemon=True, ).start() adapter = self._adapter_for_source(source) - interrupt_session_activity = getattr( - type(adapter), "interrupt_session_activity", None - ) + interrupt_session_activity = getattr(type(adapter), "interrupt_session_activity", None) if adapter and callable(interrupt_session_activity): metadata = self._thread_metadata_for_source(source) try: params = inspect.signature(interrupt_session_activity).parameters accepts_metadata = "metadata" in params or any( - param.kind is inspect.Parameter.VAR_KEYWORD - for param in params.values() + param.kind is inspect.Parameter.VAR_KEYWORD for param in params.values() ) except (TypeError, ValueError): accepts_metadata = False if accepts_metadata: - await adapter.interrupt_session_activity( - session_key, source.chat_id, metadata=metadata - ) + await adapter.interrupt_session_activity(session_key, source.chat_id, metadata=metadata) else: await adapter.interrupt_session_activity(session_key, source.chat_id) if adapter and hasattr(adapter, "get_pending_message"): @@ -573,9 +482,7 @@ class GatewayAgentCacheMixin: # message rebuilds from history; the old agent keeps its flag so a hung drain still dies. self._evict_cached_agent(session_key) - async def _refresh_agent_cache_message_count( - self, session_key: str, session_id: Optional[str] - ) -> None: + async def _refresh_agent_cache_message_count(self, session_key: str, session_id: Optional[str]) -> None: """Re-baseline a cached agent's stored message_count after THIS turn. The coherence guard compares on-disk ``message_count`` against the BUILD-time snapshot and @@ -602,24 +509,19 @@ class GatewayAgentCacheMixin: cached = _cache.get(session_key) # Only re-baseline a live 3-tuple entry; skip pending sentinels, legacy 2-tuples (they opt # out of the guard), and entries evicted/rebuilt mid-turn. - if ( - isinstance(cached, tuple) - and len(cached) > 2 - and cached[0] is not _AGENT_PENDING_SENTINEL - ): - # A snapshot taken for a different session_id (same session_key, different conversation) - # belongs to a different DB row — leave it alone. - _snapshot_sid = cached[3] if len(cached) > 3 else None - if _snapshot_sid is not None and _snapshot_sid != session_id: - return - if cached[2] != _live: - if _snapshot_sid is None: - # Legacy 3-tuple: preserve the 3-element shape for callers indexing ``cached[2]``. - _cache[session_key] = (cached[0], cached[1], _live) - else: - _cache[session_key] = ( - cached[0], cached[1], _live, _snapshot_sid, - ) + if not (isinstance(cached, tuple) and len(cached) > 2 and cached[0] is not _AGENT_PENDING_SENTINEL): + return + # A snapshot taken for a different session_id (same session_key, different conversation) + # belongs to a different DB row — leave it alone. + _snapshot_sid = cached[3] if len(cached) > 3 else None + if _snapshot_sid is not None and _snapshot_sid != session_id: + return + if cached[2] != _live: + # Legacy 3-tuple keeps its 3-element shape for callers indexing ``cached[2]``. + _cache[session_key] = ( + (cached[0], cached[1], _live) if _snapshot_sid is None + else (cached[0], cached[1], _live, _snapshot_sid) + ) def _set_pending_turn_sidecar_notes(self, session_key: str, notes: List[str]) -> None: """Stage per-turn must-deliver notes for the next agent run (one-shot).""" @@ -664,9 +566,7 @@ class GatewayAgentCacheMixin: return "[Voice channel now: not connected to a voice channel]" return f"[Voice channel now: {vc_now}]" - def _pinned_session_context_prompt( - self, context, redact_pii: bool, session_key: Optional[str] - ) -> str: + def _pinned_session_context_prompt(self, context, redact_pii: bool, session_key: Optional[str]) -> str: """Return the session-context prompt, pinned per session. Key hit → pinned bytes reused VERBATIM (immune to renderer nondeterminism); key miss → @@ -681,10 +581,7 @@ class GatewayAgentCacheMixin: return _eph_pin[1] text = build_session_context_prompt(context, redact_pii=redact_pii) if session_key: - self._session_state(session_key).conversation.ephemeral_pin = ( - _eph_key, - text, - ) + self._session_state(session_key).conversation.ephemeral_pin = (_eph_key, text) return text @staticmethod @@ -695,23 +592,22 @@ class GatewayAgentCacheMixin: rendered bytes MUST appear here — omission means a stale pinned prompt; extras only re-render. """ import hashlib - src = context.source + + def _s(v) -> str: + return str(v or "") + platform = src.platform.value if src.platform else "" discord_ids: tuple = () discord_tools = "" if src.platform == Platform.DISCORD: from gateway.session import _discord_tools_loaded - discord_tools = "1" if _discord_tools_loaded() else "0" + # message_id: only PRESENCE is rendered (the id itself arrives per-turn in the user + # message) — keying on the value would re-render every message for zero byte change. discord_ids = ( - str(src.guild_id or ""), - str(src.parent_chat_id or ""), - str(src.thread_id or ""), - str(src.chat_id or ""), - # Only PRESENCE is rendered (the id itself arrives per-turn in the user message) — - # keying on the value would re-render every message for zero byte change. + _s(src.guild_id), _s(src.parent_chat_id), _s(src.thread_id), _s(src.chat_id), "1" if src.message_id else "0", ) @@ -721,41 +617,24 @@ class GatewayAgentCacheMixin: slack_tools = "" if src.platform == Platform.SLACK: from gateway.session import _slack_tools_loaded - slack_tools = "1" if _slack_tools_loaded() else "0" try: from hermes_constants import display_hermes_home - home_display = str(display_hermes_home()) except Exception: home_display = "" key_tuple = ( - platform, - str(src.chat_id or ""), - str(src.thread_id or ""), - str(src.chat_type or ""), - str(src.chat_name or ""), - str(src.chat_topic or ""), - str(src.user_name or ""), - str(src.user_id or ""), - str(getattr(src, "profile", None) or ""), - bool(context.shared_multi_user_session), - discord_ids, - discord_tools, - slack_tools, + platform, _s(src.chat_id), _s(src.thread_id), _s(src.chat_type), _s(src.chat_name), _s(src.chat_topic), + _s(src.user_name), _s(src.user_id), _s(getattr(src, "profile", None)), + bool(context.shared_multi_user_session), discord_ids, discord_tools, slack_tools, tuple(p.value for p in context.connected_platforms), tuple( - ( - p.value, - str(getattr(hc, "name", "") or ""), - str(getattr(hc, "chat_id", "") or ""), - ) + (p.value, _s(getattr(hc, "name", "")), _s(getattr(hc, "chat_id", ""))) for p, hc in context.home_channels.items() ), - bool(redact_pii), - home_display, + bool(redact_pii), home_display, ) return hashlib.sha256(repr(key_tuple).encode("utf-8")).hexdigest() @@ -777,38 +656,53 @@ class GatewayAgentCacheMixin: _evict_state.conversation.ephemeral_pin = None _evict_state.conversation.vc_last = None + # Tests build runners with ``_agent_cache_lock = None``; evict lock-free then. _lock = getattr(self, "_agent_cache_lock", None) + _cache = getattr(self, "_agent_cache", None) evicted = None - if _lock: - with _lock: - evicted = self._agent_cache.pop(session_key, None) - else: - _cache = getattr(self, "_agent_cache", None) - if _cache is not None: + if _cache is not None: + with _lock or nullcontext(): evicted = _cache.pop(session_key, None) - agent = evicted[0] if isinstance(evicted, tuple) and evicted else evicted + agent = _first_agent(evicted) if agent is None or agent is _AGENT_PENDING_SENTINEL: return # Don't tear down an agent that's actively mid-turn — its client, # sandbox and child subagents are in use by the running request. - running_ids = self._running_agent_ids() - if id(agent) in running_ids: + if id(agent) in self._running_agent_ids(): return + self._spawn_release_thread( + self._release_evicted_agent_soft, (agent,), f"agent-evict-{str(session_key)[:24]}", inline_fallback=True, + ) + + def _spawn_release_thread(self, target, args: tuple, name: str, *, inline_fallback: bool) -> None: + """Run a release on a daemon thread; ``inline_fallback`` runs it inline when no thread can start (interpreter shutdown).""" try: - threading.Thread( - target=self._release_evicted_agent_soft, - args=(agent,), - daemon=True, - name=f"agent-evict-{str(session_key)[:24]}", - ).start() + threading.Thread(target=target, args=args, daemon=True, name=name).start() except Exception: - # If we can't spawn a thread (interpreter shutdown), release - # inline as a best-effort fallback. - with suppress(Exception): - self._release_evicted_agent_soft(agent) + if inline_fallback: + with suppress(Exception): + target(*args) + + def _finalizable_unexpired_session_entry(self, key: str): + """Return the session-store entry for ``key`` when the expiry watcher will still finalize it. + + None when the store/entry is missing, the session is not finalizable (``mode == "none"`` + never finalizes) or it has already expired (the watcher tears those down itself). + """ + _store = getattr(self, "session_store", None) + if _store is None: + return None + try: + _store._ensure_loaded() + entry = _store._entries.get(key) + except Exception: + return None + if entry is None or not _store.is_session_finalizable(entry) or _store._is_session_expired(entry): + return None + return entry def _commit_memory_before_soft_evict(self, agent: Any, key: str) -> None: """Fire on_session_end extraction before soft-evicting a live agent. @@ -817,25 +711,14 @@ class GatewayAgentCacheMixin: ``_session_expiry_watcher``'s job at true expiry. But the watcher tears down whatever it finds in ``_agent_cache``; if the LRU cap soft-evicts first, memory providers never see the transcript. So commit extraction here via ``commit_memory_session`` (no teardown). Only for - finalizable sessions — ``mode == "none"`` never finalizes. Best-effort: failures swallowed. + finalizable, not-yet-expired sessions. Best-effort: failures swallowed. """ if agent is None or not hasattr(agent, "commit_memory_session"): return if getattr(agent, "_memory_manager", None) is None: return # no external memory provider — nothing to commit try: - _store = getattr(self, "session_store", None) - if _store is None: - return - _store._ensure_loaded() - entry = _store._entries.get(key) - if entry is None: - return - # Compensate only when the watcher would expect this agent at expiry (finite policy, not yet - # expired). Expired sessions are torn down by the watcher; mode="none" is never finalized. - if not _store.is_session_finalizable(entry): - return - if _store._is_session_expired(entry): + if self._finalizable_unexpired_session_entry(key) is None: return messages = getattr(agent, "_session_messages", None) agent.commit_memory_session(messages if isinstance(messages, list) else None) @@ -867,8 +750,7 @@ class GatewayAgentCacheMixin: if hasattr(agent, "release_clients"): agent.release_clients() else: - # Older agent instance (shouldn't happen in practice) — - # fall back to the legacy full-close path. + # Older agent instance (shouldn't happen in practice) — legacy full-close path. self._cleanup_agent_resources(agent) except Exception: pass @@ -893,7 +775,6 @@ class GatewayAgentCacheMixin: bounds = getattr(self, "_agent_cache_bounds_cache", None) if bounds is None: from gateway.agent_cache_pressure import resolve_agent_cache_bounds - try: bounds = resolve_agent_cache_bounds(_load_gateway_config()) except Exception as _e: @@ -908,14 +789,12 @@ class GatewayAgentCacheMixin: def _agent_cache_cap(self) -> int: """Effective LRU cap — the configured override, else the default.""" from gateway.run import _AGENT_CACHE_MAX_SIZE - configured = self._agent_cache_bounds().max_size - return configured if configured else _AGENT_CACHE_MAX_SIZE + return self._agent_cache_bounds().max_size or _AGENT_CACHE_MAX_SIZE def _agent_cache_idle_ttl(self) -> float: """Effective idle TTL in seconds — configured override, else default.""" from gateway.run import _AGENT_CACHE_IDLE_TTL_SECS - configured = self._agent_cache_bounds().idle_ttl_secs - return configured if configured else _AGENT_CACHE_IDLE_TTL_SECS + return self._agent_cache_bounds().idle_ttl_secs or _AGENT_CACHE_IDLE_TTL_SECS def _sweep_agent_cache_under_pressure(self) -> int: """Shed cached transcripts once the gateway heap nears its budget; returns count evicted. @@ -928,9 +807,7 @@ class GatewayAgentCacheMixin: """ from gateway.run import _AGENT_PENDING_SENTINEL from gateway.agent_cache_pressure import ( - plan_pressure_evictions, - read_anon_rss_mb, - transcript_persistence_caught_up, + plan_pressure_evictions, read_anon_rss_mb, transcript_persistence_caught_up ) bounds = self._agent_cache_bounds() @@ -939,8 +816,8 @@ class GatewayAgentCacheMixin: _cache = getattr(self, "_agent_cache", None) _lock = getattr(self, "_agent_cache_lock", None) if not _cache or _lock is None: - # Nothing cached — whatever is using the heap, it isn't us, and - # warning about it every tick would point at the wrong subsystem. + # Nothing cached — whatever is using the heap, it isn't us, and warning about it every + # tick would point at the wrong subsystem. return 0 rss_mb = read_anon_rss_mb() @@ -949,22 +826,16 @@ class GatewayAgentCacheMixin: running_ids = self._running_agent_ids() + def _is_live(agent: Any) -> bool: + return agent is not None and agent is not _AGENT_PENDING_SENTINEL and id(agent) not in running_ids + def _is_evictable(key: str, agent: Any) -> bool: - if agent is None or agent is _AGENT_PENDING_SENTINEL: - return False - if id(agent) in running_ids: - return False - return transcript_persistence_caught_up(agent) + return _is_live(agent) and transcript_persistence_caught_up(agent) with _lock: - ordered = [ - (key, entry[0] if isinstance(entry, tuple) and entry else entry) - for key, entry in _cache.items() - ] + ordered = [(key, _first_agent(entry)) for key, entry in _cache.items()] plan = plan_pressure_evictions( - ordered, - is_evictable=_is_evictable, - max_evictions=bounds.max_evictions_per_pass, + ordered, is_evictable=_is_evictable, max_evictions=bounds.max_evictions_per_pass, protect_recent=bounds.protect_recent, ) for key, _ in plan: @@ -972,14 +843,7 @@ class GatewayAgentCacheMixin: if not plan: _mid_turn = sum(1 for _, a in ordered if a is not None and id(a) in running_ids) - _unflushed = sum( - 1 - for _, a in ordered - if a is not None - and a is not _AGENT_PENDING_SENTINEL - and id(a) not in running_ids - and not transcript_persistence_caught_up(a) - ) + _unflushed = sum(1 for _, a in ordered if _is_live(a) and not transcript_persistence_caught_up(a)) logger.warning( "Agent cache pressure: anon RSS %dMB over budget %dMB but no " "evictable session (%d cached, %d mid-turn, %d blocked on " @@ -997,22 +861,12 @@ class GatewayAgentCacheMixin: evicted_count = len(plan) logger.warning( - "Agent cache pressure: anon RSS %dMB over budget %dMB — evicting " - "%d LRU session(s): %s", - rss_mb, bounds.memory_high_mb, evicted_count, - ", ".join(key for key, _ in plan), + "Agent cache pressure: anon RSS %dMB over budget %dMB — evicting %d LRU session(s): %s", + rss_mb, bounds.memory_high_mb, evicted_count, ", ".join(key for key, _ in plan), ) - try: - threading.Thread( - target=self._release_pressure_batch, - args=(plan,), - daemon=True, - name="agent-cache-pressure", - ).start() - except Exception: - self._release_pressure_batch(plan) - # NOTE: _release_pressure_batch drains `plan` in place (so the trim runs with no lingering - # agent refs) — len(plan) is 0 once the daemon thread finishes, hence the pre-captured count. + self._spawn_release_thread(self._release_pressure_batch, (plan,), "agent-cache-pressure", inline_fallback=True) + # _release_pressure_batch drains `plan` in place (so the trim runs with no lingering agent + # refs) — len(plan) is 0 once the daemon thread finishes, hence the pre-captured count. return evicted_count def _release_pressure_batch(self, plan: List[tuple]) -> None: @@ -1032,7 +886,6 @@ class GatewayAgentCacheMixin: del agent try: from hermes_cli.mem_trim import trim_memory - trim_memory(force=True, reason="agent_cache_pressure") except Exception: pass @@ -1047,8 +900,8 @@ class GatewayAgentCacheMixin: _cache = getattr(self, "_agent_cache", None) if _cache is None: return - # OrderedDict.popitem(last=False) pops oldest; plain dict lacks the - # arg so skip enforcement if a test fixture swapped the cache type. + # OrderedDict.popitem(last=False) pops oldest; plain dict lacks the arg so skip enforcement + # if a test fixture swapped the cache type. if not hasattr(_cache, "move_to_end"): return @@ -1062,14 +915,12 @@ class GatewayAgentCacheMixin: cap = self._agent_cache_cap() excess = max(0, len(_cache) - cap) evict_plan: List[tuple] = [] # [(key, agent), ...] - if excess > 0: - ordered_keys = list(_cache.keys()) - for key in ordered_keys[:excess]: - entry = _cache.get(key) - agent = entry[0] if isinstance(entry, tuple) and entry else None - if agent is not None and id(agent) in running_ids: - continue # active mid-turn; don't evict, don't substitute - evict_plan.append((key, agent)) + for key in list(_cache.keys())[:excess]: + entry = _cache.get(key) + agent = entry[0] if isinstance(entry, tuple) and entry else None + if agent is not None and id(agent) in running_ids: + continue # active mid-turn; don't evict, don't substitute + evict_plan.append((key, agent)) for key, _ in evict_plan: _cache.pop(key, None) @@ -1083,19 +934,13 @@ class GatewayAgentCacheMixin: ) for key, agent in evict_plan: - logger.info( - "Agent cache at cap; evicting LRU session=%s (cache_size=%d)", - key, len(_cache), - ) + logger.info("Agent cache at cap; evicting LRU session=%s (cache_size=%d)", key, len(_cache)) if agent is not None: # Commit end-of-session memory, then soft-release, both on the daemon thread so the # (possibly network-bound) provider call never blocks the held cache lock. - threading.Thread( - target=self._commit_then_release_soft, - args=(agent, key), - daemon=True, - name=f"agent-cache-evict-{key[:24]}", - ).start() + self._spawn_release_thread( + self._commit_then_release_soft, (agent, key), f"agent-cache-evict-{key[:24]}", inline_fallback=False, + ) def _sweep_idle_cached_agents(self) -> int: """Evict cached agents idle past the idle TTL; returns the number evicted. @@ -1114,38 +959,21 @@ class GatewayAgentCacheMixin: with _lock: for key, entry in list(_cache.items()): agent = entry[0] if isinstance(entry, tuple) and entry else None - if agent is None: - continue - if id(agent) in running_ids: + if agent is None or id(agent) in running_ids: continue # mid-turn — don't tear it down last_activity = getattr(agent, "_last_activity_ts", None) - if last_activity is None: + if last_activity is None or (now - last_activity) <= idle_ttl: continue - if (now - last_activity) > idle_ttl: - # If the session hasn't actually expired in the store (e.g. daily-reset fires hours - # after the last message), keep the agent cached so the expiry watcher can still find - # it and call on_session_end() with the live transcript. BUT only defer when the - # watcher will EVER finalize it: for mode == "none" (is_session_finalizable() False) - # deferring pins the agent for the gateway's lifetime — the leak this sweep relieves. - # Those fall through to soft eviction WITHOUT on_session_end, correctly (never a - # session-end boundary). Finite sessions evicted under LRU-cap pressure are covered - # by _commit_memory_before_soft_evict on the cap path. - session_entry = None - _store = getattr(self, "session_store", None) - try: - if _store is not None: - _store._ensure_loaded() - session_entry = _store._entries.get(key) - except Exception: - session_entry = None - if ( - session_entry is not None - and _store is not None - and _store.is_session_finalizable(session_entry) - and not _store._is_session_expired(session_entry) - ): - continue # keep agent — finite session hasn't expired - to_evict.append((key, agent)) + # If the session hasn't actually expired in the store (e.g. daily-reset fires hours + # after the last message), keep the agent cached so the expiry watcher can still find + # it and call on_session_end() with the live transcript. BUT only defer when the + # watcher will EVER finalize it: for mode == "none" deferring pins the agent for the + # gateway's lifetime — the leak this sweep relieves. Those fall through to soft + # eviction WITHOUT on_session_end, correctly (never a session-end boundary). Finite + # sessions evicted under LRU-cap pressure are covered by _commit_memory_before_soft_evict. + if self._finalizable_unexpired_session_entry(key) is not None: + continue # keep agent — finite session hasn't expired + to_evict.append((key, agent)) for key, _ in to_evict: _cache.pop(key, None) for key, agent in to_evict: @@ -1153,10 +981,7 @@ class GatewayAgentCacheMixin: "Agent cache idle-TTL evict: session=%s (idle=%.0fs)", key, now - getattr(agent, "_last_activity_ts", now), ) - threading.Thread( - target=self._release_evicted_agent_soft, - args=(agent,), - daemon=True, - name=f"agent-cache-idle-{key[:24]}", - ).start() + self._spawn_release_thread( + self._release_evicted_agent_soft, (agent,), f"agent-cache-idle-{key[:24]}", inline_fallback=False, + ) return len(to_evict) diff --git a/gateway/run_busy.py b/gateway/run_busy.py index 53946c7587..a332b66504 100644 --- a/gateway/run_busy.py +++ b/gateway/run_busy.py @@ -1,8 +1,8 @@ -"""Busy-session queueing, slot claims, slash dispatch tables and destructive-slash confirmation for GatewayRunner. +"""Busy-session queueing, slot claims, slash dispatch tables and destructive-slash confirmation +for GatewayRunner (mixin bound via the MRO). -Split out of ``gateway/run.py``; bound onto ``GatewayRunner`` via the MRO. -``gateway.run`` internals are imported lazily inside method bodies (import cycle), -so ``patch("gateway.run.X")`` keeps intercepting them at call time. +``gateway.run`` internals are imported lazily inside method bodies (import cycle), so +``patch("gateway.run.X")`` keeps intercepting them at call time. """ from __future__ import annotations @@ -10,6 +10,7 @@ from __future__ import annotations import logging from typing import TYPE_CHECKING import asyncio +import contextlib import json import os import time @@ -27,105 +28,83 @@ logger = logging.getLogger("gateway.run") class GatewayBusySessionMixin: - """Busy-session queueing, slot claims, slash dispatch tables and destructive-slash confirmation for GatewayRunner.""" + """Busy-session queueing, slot claims, slash dispatch tables, destructive-slash confirmation.""" - def _queue_during_drain_enabled( - self, busy_input_mode: Optional[str] = None - ) -> bool: - # "queue" and "steer" both mean messages must not be lost across restart: queue them for - # the newly-spawned gateway process to pick up. "interrupt" mode drops them. + def _queue_during_drain_enabled(self, busy_input_mode: Optional[str] = None) -> bool: + # "queue"/"steer" mean messages survive a restart (queued for the new process); "interrupt" drops. mode = busy_input_mode or self._busy_input_mode return self._restart_requested and mode in {"queue", "steer"} + def _overflow_queue(self, session_key: str): + """The session's FIFO overflow list, or None when no session state exists yet.""" + state = self._peek_session_state(session_key) + return state.conversation.queued_events if state else None + def _enqueue_fifo(self, session_key: str, queued_event: "MessageEvent", adapter: Any) -> None: """Append a /queue event to the FIFO chain for a session.""" - if adapter is None: - return - pending_slot = getattr(adapter, "_pending_messages", None) + pending_slot = getattr(adapter, "_pending_messages", None) if adapter is not None else None if pending_slot is None: return if session_key in pending_slot: - self._session_state(session_key).conversation.queued_events.append( - queued_event - ) + self._session_state(session_key).conversation.queued_events.append(queued_event) else: pending_slot[session_key] = queued_event def _promote_queued_event( - self, - session_key: str, - adapter: Any, - pending_event: Optional["MessageEvent"], + self, session_key: str, adapter: Any, pending_event: Optional["MessageEvent"] ) -> Optional["MessageEvent"]: - """Promote the next overflow item after the slot was drained. + """Promote the next overflow item after the slot drained. - If pending_event is None, return the overflow head as the new pending_event; if the slot is - already populated (interrupt follow-up etc.), stage the head there for the NEXT recursion. - Returns the (possibly updated) pending_event. + ``pending_event`` None → the overflow head becomes the pending event; otherwise the head is + staged into the slot for the NEXT recursion. Returns the (possibly updated) pending_event. """ - _q_state = self._peek_session_state(session_key) - overflow = _q_state.conversation.queued_events if _q_state else None + overflow = self._overflow_queue(session_key) if not overflow: return pending_event - next_queued = overflow.pop(0) if pending_event is None: - return next_queued + return overflow.pop(0) if adapter is not None and hasattr(adapter, "_pending_messages"): - adapter._pending_messages[session_key] = next_queued - else: - # No adapter — push back so we don't silently drop the item. - overflow.insert(0, next_queued) + adapter._pending_messages[session_key] = overflow.pop(0) + # else: no adapter — leave the head in place so we don't silently drop it. return pending_event def _queue_depth(self, session_key: str, *, adapter: Any = None) -> int: """Total pending /queue items for a session — slot + overflow.""" - _q_state = self._peek_session_state(session_key) - depth = len(_q_state.conversation.queued_events) if _q_state else 0 + depth = len(self._overflow_queue(session_key) or ()) if adapter is not None and session_key in getattr(adapter, "_pending_messages", {}): depth += 1 return depth - def _rescue_orphaned_overflow( - self, session_key: str, adapter: Any - ) -> Optional["MessageEvent"]: - """Pop the oldest orphaned FIFO overflow event for an idle session. + def _rescue_orphaned_overflow(self, session_key: str, adapter: Any) -> Optional["MessageEvent"]: + """Pop the oldest orphaned FIFO overflow event for an idle session (None if nothing to rescue). - ``queued_events`` drains only at the post-turn promotion site in ``_run_agent``; if a busy - window ends without that drain (early recursion exit, exception/interrupt/generation-bump), - the overflow is silently orphaned. Called when a NEW event arrives for a NON-busy session: - the oldest orphan is returned to run as THIS turn, the next is staged into the slot so the - chain continues in arrival order, and the caller enqueues the incoming event behind it. The - returned event is REMOVED from both stores, else the post-turn dequeue would run it twice. - Returns ``None`` when there is nothing to rescue (no overflow, slot occupied, or no slot). + ``queued_events`` drains only at the post-turn promotion site; a busy window ending without + it (early exit, exception/interrupt/generation-bump) orphans the overflow. On a NEW event for + a NON-busy session the oldest orphan runs as THIS turn, the next is staged into the slot so + arrival order holds, and the caller enqueues the incoming event behind it. The returned + event is REMOVED from both stores, else the post-turn dequeue would run it twice. """ try: - _q_state = self._peek_session_state(session_key) - overflow = _q_state.conversation.queued_events if _q_state else None + overflow = self._overflow_queue(session_key) if not overflow: return None pending_slot = getattr(adapter, "_pending_messages", None) if not isinstance(pending_slot, dict) or pending_slot.get(session_key): - # Slot occupied (busy) or no slot storage — promotion owns - # this; do not fight it from the idle path. - return None + return None # slot occupied (busy) or no slot storage — promotion owns this head = overflow.pop(0) - # Keep the slot occupied for the rest of the chain so the drain promotes in order and - # any mid-chain arrival routes to overflow instead of jumping the queue (same invariant - # as the drain's own _promote_queued_event). Only ONE event fits the slot. + # Keep the slot occupied so the drain promotes in order and a mid-chain arrival routes + # to overflow instead of jumping the queue (same invariant as _promote_queued_event). if overflow: pending_slot[session_key] = overflow.pop(0) logger.warning( "Rescued orphaned FIFO overflow event for idle session " "%s — it was queued during a busy window but the post-turn " - "drain never promoted it (#99882)", - session_key, + "drain never promoted it (#99882)", session_key, ) if overflow: logger.warning( "%d overflow event(s) still queued for session %s after " - "rescue staging (will drain via normal promotion)", - len(overflow), - session_key, + "rescue staging (will drain via normal promotion)", len(overflow), session_key, ) return head except Exception: @@ -134,20 +113,12 @@ class GatewayBusySessionMixin: @staticmethod def _is_goal_continuation_event(event_or_text: Any) -> bool: - """Return True for synthetic /goal continuation turns. - - Goal continuations are normal queued user-role events, so pause/clear must distinguish - them from real user /queue messages before removing or suppressing them. - """ + """True for synthetic /goal continuation turns (so pause/clear can spare real /queue items).""" text = getattr(event_or_text, "text", event_or_text) or "" return str(text).startswith("[Continuing toward your standing goal]\nGoal:") def _clear_goal_pending_continuations(self, session_key: str, adapter: Any) -> int: - """Remove queued synthetic /goal continuations for one session. - - User /goal pause/clear can race a judge-queued continuation; only synthetic goal - continuations are removed, normal /queue and user follow-up events are preserved. - """ + """Remove queued synthetic /goal continuations for one session; real /queue items are kept.""" removed = 0 pending_slot = getattr(adapter, "_pending_messages", None) if adapter is not None else None if isinstance(pending_slot, dict): @@ -156,16 +127,11 @@ class GatewayBusySessionMixin: pending_slot.pop(session_key, None) removed += 1 - _q_state = self._peek_session_state(session_key) - overflow = _q_state.conversation.queued_events if _q_state else [] + overflow = self._overflow_queue(session_key) if overflow: - kept = [] - for queued_event in overflow: - if self._is_goal_continuation_event(queued_event): - removed += 1 - else: - kept.append(queued_event) - _q_state.conversation.queued_events = kept + kept = [e for e in overflow if not self._is_goal_continuation_event(e)] + removed += len(overflow) - len(kept) + self._peek_session_state(session_key).conversation.queued_events = kept return removed def _goal_still_active_for_session(self, session_id: str) -> bool: @@ -183,7 +149,6 @@ class GatewayBusySessionMixin: """Return the configured active chat session cap, if enabled.""" try: from hermes_cli.active_sessions import resolve_max_concurrent_sessions - return resolve_max_concurrent_sessions(getattr(self, "config", None)) except Exception: return None @@ -191,31 +156,25 @@ class GatewayBusySessionMixin: def _active_session_limit_message(self, session_key: str) -> Optional[str]: """Return a user-facing rejection when starting a new session exceeds the cap.""" max_sessions = self._get_max_concurrent_sessions() - if max_sessions is None: - return None - if self._is_session_running(session_key): + if max_sessions is None or self._is_session_running(session_key): return None active_count = self._running_agent_count() if active_count < max_sessions: return None from hermes_cli.active_sessions import active_session_limit_message - return active_session_limit_message(active_count, max_sessions) def _claim_active_session_slot( - self, - session_key: str, - source: SessionSource, + self, session_key: str, source: SessionSource ) -> tuple[Any, Optional[str]]: """Claim a cross-process active-session slot for a new gateway turn.""" if self._is_session_running(session_key): return None, None - local_limit_message = self._active_session_limit_message(session_key) - if local_limit_message is not None: - return None, local_limit_message + limit_message = self._active_session_limit_message(session_key) + if limit_message is not None: + return None, limit_message try: from hermes_cli.active_sessions import try_acquire_active_session - platform = source.platform.value if source and source.platform else "gateway" return try_acquire_active_session( session_id=session_key, @@ -225,9 +184,8 @@ class GatewayBusySessionMixin: "platform": platform, "chat_id": getattr(source, "chat_id", "") or "", "user_id": getattr(source, "user_id", "") or "", - # Writer identity for re-entrancy: if this process leaks a lease for this session - # (exception path skipped release), the next turn re-acquires its own entry rather - # than being fenced out forever — pruning only reclaims entries whose PROCESS died. + # Writer identity: a leaked lease from this process is re-acquired by the next + # turn rather than fencing it out forever (pruning only reclaims dead PROCESSES). "live_session_id": str(session_key), }, ) @@ -237,43 +195,36 @@ class GatewayBusySessionMixin: @staticmethod def _agent_has_active_subagents(running_agent: Any) -> bool: - """Return True when *running_agent* is driving subagents via ``delegate_task``. - - ``AIAgent.interrupt()`` cascades through ``_active_children`` and aborts in-flight subagent - work, so callers demote ``busy_input_mode='interrupt'`` to ``queue`` while this is True; - explicit ``/stop`` is untouched. Fail-safe: returns False on any attribute/lock error. - """ + """True when *running_agent* is driving subagents (callers demote interrupt → queue; + ``interrupt()`` would cascade through ``_active_children``). Fail-safe False on any error.""" from gateway.run import _AGENT_PENDING_SENTINEL if running_agent is None or running_agent is _AGENT_PENDING_SENTINEL: return False children = getattr(running_agent, "_active_children", None) - # AIAgent always initialises this as a concrete list. Reject anything that isn't a real - # collection — guards against ``MagicMock()._active_children`` auto-creating a truthy stub - # in tests and triggering the demotion for an agent with no subagents. - if not isinstance(children, (list, tuple, set)): - return False - if not children: + # Real collections only — a ``MagicMock()._active_children`` auto-attr must not demote. + if not isinstance(children, (list, tuple, set)) or not children: return False lock = getattr(running_agent, "_active_children_lock", None) try: - if lock is not None: - with lock: - return bool(children) - return bool(children) + with lock if lock is not None else contextlib.nullcontext(): + return bool(children) except Exception: return False async def _session_has_compression_in_flight(self, session_key: str) -> bool: - """Return True when a compression lock is held for this session's id. - - Gateway ``interrupt`` busy mode could start a follow-up against the pre-rotation parent while - compression is mid-flight, producing orphaned compression siblings; callers demote interrupt - to queue when True. Both blocking sources (``session_store`` lock + JSON load, SQLite lock - holder SELECT) run in a worker thread so a large state.db never freezes the event loop. - """ + """True when a compression lock is held for this session's id (callers demote interrupt → + queue, else a follow-up against the pre-rotation parent orphans compression siblings). + Both blocking reads run in a worker thread so a large state.db never freezes the loop.""" session_store = getattr(self, "session_store", None) if not session_key or session_store is None: return False + def _assume_active(what: str, ident) -> bool: + logger.warning( + "Compression in-flight check failed while reading %s %s; treating compression as " + "active to avoid interrupting a possible parent-session rotation", what, ident, exc_info=True, + ) + return True + try: session_id = await asyncio.to_thread( self._lookup_session_id_under_store_lock, session_store, session_key @@ -281,38 +232,20 @@ class GatewayBusySessionMixin: except (AttributeError, TypeError): return False except Exception: - logger.warning( - "Compression in-flight check failed while reading session %s; " - "treating compression as active to avoid interrupting a possible " - "parent-session rotation", - session_key, - exc_info=True, - ) - return True - if not session_id: - return False + return _assume_active("session", session_key) session_db = getattr(self, "_session_db", None) - if session_db is None: + if not session_id or session_db is None: return False raw_db = getattr(session_db, "_db", session_db) try: - holder = await asyncio.to_thread( - raw_db.get_compression_lock_holder, str(session_id) - ) + holder = await asyncio.to_thread(raw_db.get_compression_lock_holder, str(session_id)) # Production returns Optional[str]. Reject non-strings so a MagicMock auto-attr (or any # unexpected truthy) cannot look like a held lock and skip hygiene. return isinstance(holder, str) and bool(holder) except (AttributeError, TypeError): return False except Exception: - logger.warning( - "Compression in-flight check failed while reading lock holder " - "for session %s; treating compression as active to avoid " - "interrupting a possible parent-session rotation", - session_id, - exc_info=True, - ) - return True + return _assume_active("lock holder for session", session_id) @staticmethod def _lookup_session_id_under_store_lock(session_store, session_key: str): @@ -323,23 +256,21 @@ class GatewayBusySessionMixin: entry = session_store._entries.get(session_key) # noqa: SLF001 return getattr(entry, "session_id", None) if entry is not None else None + # Metadata that must match for two pending events to merge into one slot. + _SECURITY_METADATA_KEYS = ( + "hermes_plugin_id", "hermes_plugin_injection", "gateway_session_key", + "gateway_session_id", "gateway_session_strict", + ) + def _queue_or_replace_pending_event(self, session_key: str, event: MessageEvent) -> None: from gateway.run import merge_pending_message_event adapter = self._adapter_for_source(event.source) if not adapter: return - # Route through the ``/queue`` FIFO infrastructure so each follow-up gets its own turn in - # arrival order (merge_text=False silently OVERWROTE the single pending slot). Photo bursts - # still merge into the head slot (album semantics); everything else appends to the tail. + # FIFO so each follow-up gets its own turn in arrival order (the single pending slot used to + # be silently OVERWRITTEN). Photo bursts still merge into the head slot (album semantics). pending_slot = getattr(adapter, "_pending_messages", None) existing = pending_slot.get(session_key) if isinstance(pending_slot, dict) else None - security_metadata_keys = ( - "hermes_plugin_id", - "hermes_plugin_injection", - "gateway_session_key", - "gateway_session_id", - "gateway_session_strict", - ) same_security_context = existing is not None and ( getattr(existing, "internal", False) == getattr(event, "internal", False) and getattr(existing, "allow_gateway_control", True) @@ -347,7 +278,7 @@ class GatewayBusySessionMixin: and all( (getattr(existing, "metadata", None) or {}).get(key) == (getattr(event, "metadata", None) or {}).get(key) - for key in security_metadata_keys + for key in self._SECURITY_METADATA_KEYS ) ) if same_security_context and ( @@ -358,9 +289,7 @@ class GatewayBusySessionMixin: ): # Preserve photo-burst / media-merge semantics for the head slot. merge_pending_message_event( - adapter._pending_messages, - session_key, - event, + adapter._pending_messages, session_key, event, merge_text=event.message_type == MessageType.TEXT, ) return @@ -368,37 +297,26 @@ class GatewayBusySessionMixin: if self._queue_depth(session_key, adapter=adapter) >= self._BUSY_QUEUE_MAX_PENDING: logger.warning( "Dropping busy-mode follow-up for session %s — pending queue at cap (%d).", - session_key, - self._BUSY_QUEUE_MAX_PENDING, + session_key, self._BUSY_QUEUE_MAX_PENDING, ) return self._enqueue_fifo(session_key, event, adapter) async def _prepare_busy_steer_text(self, event: MessageEvent) -> str: - """Return steerable text for a busy follow-up, transcribing voice first. + """Steerable text for a busy follow-up, transcribing voice-message media first. - Successful steer messages bypass the inbound STT queue, so without this a media-only voice - follow-up has empty text and steer silently degrades to queue mode. Only voice-message media - (not audio file attachments) is transcribed; on failure keep any caption and let the steer - fallback handle it. Goes through ``_transcribe_and_echo_pending_voice`` — the single - out-of-band STT choke point — so STT runs at most once per message (cached on the event). + Steer bypasses the inbound STT queue, so a media-only voice follow-up would otherwise + silently degrade to queue mode. Uses the single out-of-band STT choke point, so STT runs at + most once per message; on failure the caption (if any) is kept. """ text = (event.text or "").strip() if not self._pending_event_audio_paths(event): return text - - adapter = self._adapter_for_source(event.source) enriched_text, successful_transcripts = await self._transcribe_and_echo_pending_voice( - event, - adapter, - event.source, - text, - log_context="Busy-steer", + event, self._adapter_for_source(event.source), event.source, text, log_context="Busy-steer" ) - if not successful_transcripts: - return text - return (enriched_text or text).strip() + return (enriched_text or text).strip() if successful_transcripts else text @staticmethod def _busy_reply_to(event: MessageEvent, reply_anchor): @@ -411,64 +329,57 @@ class GatewayBusySessionMixin: else (None if event.source.platform == Platform.TELEGRAM and event.source.thread_id else event.message_id) ) + async def _send_busy_reply(self, event: MessageEvent, adapter, content: str, *, plain_anchor: bool = False) -> None: + """Send a busy-path reply anchored to the event (thread metadata included).""" + reply_anchor = self._reply_anchor_for_event(event) + await adapter._send_with_retry( + chat_id=event.source.chat_id, content=content, + reply_to=reply_anchor if plain_anchor else self._busy_reply_to(event, reply_anchor), + metadata=self._thread_metadata_for_source(event.source, reply_anchor), + ) + async def _send_busy_drain_notice(self, event: MessageEvent, session_key: str, effective_mode: str) -> None: """Busy path while the gateway is restarting/stopping: queue (if allowed) and tell the user.""" adapter = self._adapter_for_source(event.source) if not adapter: return - - reply_anchor = self._reply_anchor_for_event(event) - thread_meta = self._thread_metadata_for_source(event.source, reply_anchor) if self._queue_during_drain_enabled(effective_mode): self._queue_or_replace_pending_event(session_key, event) message = f"⏳ Gateway {self._status_action_gerund()} — queued for the next turn after it comes back." else: message = f"⏳ Gateway is {self._status_action_gerund()} and is not accepting another turn right now." + await self._send_busy_reply(event, adapter, message) - await adapter._send_with_retry( - chat_id=event.source.chat_id, - content=message, - reply_to=self._busy_reply_to(event, reply_anchor), - metadata=thread_meta, - ) + # Bare-word approval replies → (verb, args) for the synthesized slash command. + _PLAINTEXT_APPROVAL_WORDS: Dict[str, tuple] = { + **{w: ("approve", "") for w in ("approve", "yes", "ok", "okay", "confirm", "y", "👍")}, + **{w: ("deny", "") for w in ("deny", "no", "reject", "cancel", "n", "👎")}, + **{w: ("approve", "always") for w in ("always", "approve always", "always approve")}, + **{w: ("approve", "session") for w in ("session", "approve session", "session approve")}, + } async def _route_plaintext_approval_while_busy(self, event: MessageEvent, session_key: str) -> bool: """Route a bare "yes"/"no" to the approval handlers while a dangerous-command approval blocks. Returns True when the message was consumed as an approval response. """ - # Approval routing: while blocked on a dangerous-command approval, a bare "yes" must reach the - # approval handler, not be steered/queued/interrupted (else it queues behind a turn that can't - # start until the approval resolves -> auto-deny deadlock). Slash forms already bypass at the - # base-adapter guard. Gated on has_blocking_approval so a conversational "yes" never fires a - # command. Reuse the /approve and /deny handlers; the busy path does not auto-send their return. + # A bare "yes" while blocked on a dangerous-command approval must reach the approval handler, + # not queue behind a turn that can't start until it resolves (auto-deny deadlock). Gated on + # has_blocking_approval so a conversational "yes" never fires a command. try: from tools.approval import has_blocking_approval if event.allow_gateway_control and has_blocking_approval(session_key): _raw_text = (event.text or "").strip().lower() - _approve_words = {"approve", "yes", "ok", "okay", "confirm", "y", "👍"} - _deny_words = {"deny", "no", "reject", "cancel", "n", "👎"} - _approval_handler = None - _normalized_args = "" - if _raw_text in _approve_words: - _approval_handler = self._handle_approve_command - elif _raw_text in _deny_words: - _approval_handler = self._handle_deny_command - elif _raw_text in {"always", "approve always", "always approve"}: - _approval_handler = self._handle_approve_command - _normalized_args = "always" - elif _raw_text in {"session", "approve session", "session approve"}: - _approval_handler = self._handle_approve_command - _normalized_args = "session" - if _approval_handler is not None: + _match = self._PLAINTEXT_APPROVAL_WORDS.get(_raw_text) + if _match is not None: + _verb, _normalized_args = _match + _approval_handler = ( + self._handle_approve_command if _verb == "approve" else self._handle_deny_command + ) # Synthesize "/approve [args]" / "/deny" so the slash handlers parse modifiers via # event.get_command_args(). Always a literal "/": is_command()/get_command_args() # don't recognize per-platform display prefixes ("!" on Slack/Matrix). - _verb = "approve" if _approval_handler is self._handle_approve_command else "deny" - _synth = f"/{_verb}" - if _normalized_args: - _synth = f"{_synth} {_normalized_args}" - event.text = _synth + event.text = f"/{_verb} {_normalized_args}".rstrip() _reply = await _approval_handler(event) logger.info( "Approval response via plain text: session=%s verb=%s args=%r", @@ -478,113 +389,76 @@ class GatewayBusySessionMixin: if _adapter and _reply: _text, _eph_ttl = _adapter._unwrap_ephemeral(_reply) if _text: - _anchor = self._reply_anchor_for_event(event) - await _adapter._send_with_retry( - chat_id=event.source.chat_id, - content=_text, - reply_to=_anchor, - metadata=self._thread_metadata_for_source(event.source, _anchor), - ) + await self._send_busy_reply(event, _adapter, _text, plain_anchor=True) return True except Exception: logger.warning( "Plain-text approval routing failed for session %s; " - "falling through to busy handling", - session_key, exc_info=True, + "falling through to busy handling", session_key, exc_info=True, ) return False async def _resolve_busy_steer_or_redirect( - self, - event: MessageEvent, - session_key: str, - effective_mode: str, - running_agent: Any, + self, event: MessageEvent, session_key: str, effective_mode: str, running_agent: Any ) -> "GatewayRunner._BusySteerOutcome": """Apply interrupt->queue demotions, then attempt steer (steer mode) or redirect (interrupt mode).""" from gateway.run import _AGENT_PENDING_SENTINEL - # Steer mode injects mid-run via running_agent.steer(); fall back to queue (nothing lost) if the - # agent isn't running yet (sentinel), lacks steer(), or the payload is empty. interrupt() - # cascades to ``_active_children`` and aborts delegate_task work, so demote ``interrupt`` to - # ``queue`` while the parent drives subagents; explicit /stop and /new still force-cancel all. + # Steer injects mid-run via running_agent.steer(), falling back to queue (nothing lost) when + # the agent isn't running yet, lacks steer(), or the payload is empty. Interrupt is demoted + # to queue while subagents run (interrupt() would abort them); /stop and /new still cancel all. demoted_for_subagents = ( - effective_mode == "interrupt" - and self._agent_has_active_subagents(running_agent) + effective_mode == "interrupt" and self._agent_has_active_subagents(running_agent) ) if demoted_for_subagents: - logger.info( - "Demoting busy_input_mode 'interrupt' to 'queue' for session %s " - "because the running agent has active subagents (#30170)", - session_key, - ) - effective_mode = "queue" + effective_mode = self._demote_interrupt(session_key, "the running agent has active subagents (#30170)") demoted_for_compression = ( - effective_mode == "interrupt" - and await self._session_has_compression_in_flight(session_key) + effective_mode == "interrupt" and await self._session_has_compression_in_flight(session_key) ) if demoted_for_compression: - logger.info( - "Demoting busy_input_mode 'interrupt' to 'queue' for session %s " - "because context compression is in flight (#56391)", - session_key, - ) - effective_mode = "queue" - steered = False - redirected = False + effective_mode = self._demote_interrupt(session_key, "context compression is in flight (#56391)") + steered = redirected = False + agent_live = running_agent is not None and running_agent is not _AGENT_PENDING_SENTINEL + plain_text = ( + event.message_type == MessageType.TEXT and not event.media_urls and not event.media_types + ) if effective_mode == "steer": steer_text = await self._prepare_busy_steer_text(event) - # Steerable: plain text, OR every attachment is STT-eligible voice media whose transcript - # was folded into steer_text — else a voice note in steer mode silently degrades to queue. + # Steerable: plain text, OR every attachment is voice media folded into steer_text. _steer_media_urls = getattr(event, "media_urls", None) or [] _steer_all_voice = bool(_steer_media_urls) and ( len(self._pending_event_audio_paths(event)) == len(_steer_media_urls) ) - can_steer = ( - steer_text - and ( - ( - event.message_type == MessageType.TEXT - and not event.media_urls - and not event.media_types - ) - or _steer_all_voice - ) - and running_agent is not None - and running_agent is not _AGENT_PENDING_SENTINEL - and hasattr(running_agent, "steer") - ) - if can_steer: - try: - steered = bool(running_agent.steer(steer_text)) - except Exception as exc: - logger.warning("Gateway steer failed for session %s: %s", session_key, exc) - steered = False + if steer_text and (plain_text or _steer_all_voice) and agent_live and hasattr(running_agent, "steer"): + steered = self._try_agent_verb(running_agent, "steer", steer_text, session_key) if not steered: - # Fall back to queue (merge into pending messages, no interrupt) effective_mode = "queue" elif ( - effective_mode == "interrupt" - and event.message_type == MessageType.TEXT - and not event.media_urls - and not event.media_types - and running_agent is not None - and running_agent is not _AGENT_PENDING_SENTINEL + effective_mode == "interrupt" and plain_text and agent_live and getattr(running_agent, "_supports_active_turn_redirect", False) is True and hasattr(running_agent, "redirect") ): - try: - redirected = bool(running_agent.redirect((event.text or "").strip())) - except Exception as exc: - logger.warning("Gateway redirect failed for session %s: %s", session_key, exc) - redirected = False + redirected = self._try_agent_verb( + running_agent, "redirect", (event.text or "").strip(), session_key + ) return self._BusySteerOutcome( - effective_mode=effective_mode, - demoted_for_subagents=demoted_for_subagents, - demoted_for_compression=demoted_for_compression, - steered=steered, - redirected=redirected, + effective_mode=effective_mode, demoted_for_subagents=demoted_for_subagents, + demoted_for_compression=demoted_for_compression, steered=steered, redirected=redirected, ) + @staticmethod + def _demote_interrupt(session_key: str, why: str) -> str: + logger.info("Demoting busy_input_mode 'interrupt' to 'queue' for session %s because %s", session_key, why) + return "queue" + + @staticmethod + def _try_agent_verb(running_agent, verb: str, text: str, session_key: str) -> bool: + """Call ``running_agent.(text)`` (steer/redirect); False + warning on failure.""" + try: + return bool(getattr(running_agent, verb)(text)) + except Exception as exc: + logger.warning("Gateway %s failed for session %s: %s", verb, session_key, exc) + return False + async def _interrupt_running_agent_for_busy_event(self, event: MessageEvent, adapter, running_agent) -> None: """Interrupt mode: abort in-flight tool calls; the agent loop exits at its next check point.""" from gateway.run import _build_media_placeholder @@ -593,11 +467,7 @@ class GatewayBusySessionMixin: _media_urls = getattr(event, "media_urls", None) or [] if self._pending_event_audio_paths(event): _interrupt_text, _ = await self._transcribe_and_echo_pending_voice( - event, - adapter, - event.source, - event.text or "", - log_context="Voice-busy-interrupt", + event, adapter, event.source, event.text or "", log_context="Voice-busy-interrupt", ) elif not _interrupt_text and _media_urls: _interrupt_text = _build_media_placeholder(event) @@ -606,244 +476,161 @@ class GatewayBusySessionMixin: pass # don't let interrupt failure block the ack def _busy_steer_ack_enabled(self, event: MessageEvent, session_key: str) -> bool: - # Steer mode already injected the text; some mobile chat setups want silent steering (like STT - # echo suppression) — keep the behavior, drop only the confirmation bubble. + # Some mobile chat setups want silent steering — keep the behavior, drop the bubble. from gateway.run import _load_gateway_config, _platform_config_key from gateway.display_config import resolve_display_setting - platform_key = _platform_config_key(event.source.platform) steer_ack_env = os.environ.get("HERMES_GATEWAY_BUSY_STEER_ACK_ENABLED") if steer_ack_env is not None: steer_ack_enabled = steer_ack_env.strip().lower() in {"1", "true", "yes", "on"} else: steer_ack_enabled = bool( resolve_display_setting( - _load_gateway_config(), - platform_key, - "busy_steer_ack_enabled", - True, + _load_gateway_config(), _platform_config_key(event.source.platform), + "busy_steer_ack_enabled", True, ) ) if not steer_ack_enabled: logger.debug("Busy steer ack suppressed for session %s", session_key) return steer_ack_enabled + _BUSY_DEMOTED_TAIL = ( + " — your message is queued for when it finishes (use /stop to cancel everything)." + ) + def _compose_busy_ack_message( - self, - event: MessageEvent, - now: float, - _busy_state, - running_agent: Any, - *, - is_steer_mode: bool, - is_queue_mode: bool, - is_redirect_mode: bool, - demoted_for_subagents: bool, - demoted_for_compression: bool, + self, event: MessageEvent, now: float, _busy_state, running_agent: Any, *, + is_steer_mode: bool, is_queue_mode: bool, is_redirect_mode: bool, + demoted_for_subagents: bool, demoted_for_compression: bool, ) -> str: from gateway.run import ( - _AGENT_PENDING_SENTINEL, - _hermes_home, - _load_gateway_config, - _platform_config_key, + _AGENT_PENDING_SENTINEL, _hermes_home, _load_gateway_config, _platform_config_key ) from gateway.display_config import resolve_display_setting - # Mobile chat defaults keep the ack terse; iteration/tool detail stays in logs and can be opted - # in per platform via display.platforms..busy_ack_detail. + # Terse by default; iteration/tool detail opts in via display.platforms.

.busy_ack_detail. status_parts = [] busy_ack_detail_enabled = bool( resolve_display_setting( - _load_gateway_config(), - _platform_config_key(event.source.platform), - "busy_ack_detail", - True, + _load_gateway_config(), _platform_config_key(event.source.platform), + "busy_ack_detail", True, ) ) - if busy_ack_detail_enabled and running_agent and running_agent is not _AGENT_PENDING_SENTINEL: try: summary = running_agent.get_activity_summary() - iteration = summary.get("api_call_count", 0) - max_iter = summary.get("max_iterations", 0) - current_tool = summary.get("current_tool") - start_ts = _busy_state.turn.started_ts if _busy_state else 0 - if start_ts: - elapsed_min = int((now - start_ts) / 60) - if elapsed_min > 0: - status_parts.append(f"{elapsed_min} min elapsed") - if max_iter: - status_parts.append(f"iteration {iteration}/{max_iter}") - if current_tool: - status_parts.append(f"running: {current_tool}") + elapsed_min = 0 + if _busy_state and _busy_state.turn.started_ts: + elapsed_min = int((now - _busy_state.turn.started_ts) / 60) + if elapsed_min > 0: + status_parts.append(f"{elapsed_min} min elapsed") + if summary.get("max_iterations", 0): + status_parts.append( + f"iteration {summary.get('api_call_count', 0)}/{summary.get('max_iterations', 0)}" + ) + if summary.get("current_tool"): + status_parts.append(f"running: {summary.get('current_tool')}") except Exception: pass - status_detail = f" ({', '.join(status_parts)})" if status_parts else "" if is_steer_mode: - message = ( - f"⏩ Steered into current run{status_detail}. " - f"Your message arrives after the next tool call." - ) + head, tail = "⏩ Steered into current run", ". Your message arrives after the next tool call." elif is_redirect_mode: - message = ( - f"↪ Redirected current run{status_detail}. " - f"I'll adjust using your correction." - ) + head, tail = "↪ Redirected current run", ". I'll adjust using your correction." elif is_queue_mode and demoted_for_subagents: # Explain the demotion: the follow-up didn't kill the subagent; /stop is the escape hatch. - message = ( - f"⏳ Subagent working{status_detail} — your message is queued for " - f"when it finishes (use /stop to cancel everything)." - ) + head, tail = "⏳ Subagent working", self._BUSY_DEMOTED_TAIL elif is_queue_mode and demoted_for_compression: - message = ( - f"⏳ Compressing context{status_detail} — your message is queued for " - f"when it finishes (use /stop to cancel everything)." - ) + head, tail = "⏳ Compressing context", self._BUSY_DEMOTED_TAIL elif is_queue_mode: - message = ( - f"⏳ Queued for the next turn{status_detail}. " - f"I'll respond once the current task finishes." - ) + head, tail = "⏳ Queued for the next turn", ". I'll respond once the current task finishes." else: - message = ( - f"⚡ Interrupting current task{status_detail}. " - f"I'll respond to your message shortly." - ) + head, tail = "⚡ Interrupting current task", ". I'll respond to your message shortly." + message = f"{head}{status_detail}{tail}" - # First-touch onboarding: one-time hint about the queue/interrupt knob; the flag is persisted to - # config.yaml so it never fires again on this install. + # One-time onboarding hint about the queue/interrupt knob (flag persisted to config.yaml). try: - from agent.onboarding import ( - BUSY_INPUT_FLAG, - busy_input_hint_gateway, - is_seen, - mark_seen, - ) - _user_cfg = _load_gateway_config() - if not is_seen(_user_cfg, BUSY_INPUT_FLAG): - if is_steer_mode: - _hint_mode = "steer" - elif is_queue_mode: - _hint_mode = "queue" - elif is_redirect_mode: - _hint_mode = "redirect" - else: - _hint_mode = "interrupt" - message = ( - f"{message}\n\n" - f"{busy_input_hint_gateway(_hint_mode)}" + from agent.onboarding import (BUSY_INPUT_FLAG, busy_input_hint_gateway, is_seen, mark_seen) + if not is_seen(_load_gateway_config(), BUSY_INPUT_FLAG): + _hint_mode = ( + "steer" if is_steer_mode + else "queue" if is_queue_mode + else "redirect" if is_redirect_mode + else "interrupt" ) + message = f"{message}\n\n{busy_input_hint_gateway(_hint_mode)}" mark_seen(_hermes_home / "config.yaml", BUSY_INPUT_FLAG) except Exception as _onb_err: logger.debug("Failed to apply busy-input onboarding hint: %s", _onb_err) return message async def _send_busy_ack_reply(self, event: MessageEvent, adapter, message: str) -> None: - reply_anchor = self._reply_anchor_for_event(event) - thread_meta = self._thread_metadata_for_source(event.source, reply_anchor) try: - await adapter._send_with_retry( - chat_id=event.source.chat_id, - content=message, - reply_to=self._busy_reply_to(event, reply_anchor), - metadata=thread_meta, - ) + await self._send_busy_reply(event, adapter, message) except Exception as e: logger.debug("Failed to send busy-ack: %s", e) async def _handle_active_session_busy_message(self, event: MessageEvent, session_key: str) -> bool: - # Authorization gate: the cold path (_handle_message) checks _is_user_authorized before - # creating a session; the busy path must enforce the same check, else unauthorized users in - # shared threads (Slack/Telegram/Discord) inject messages into a session they don't own. + # Same authorization gate as the cold path, else unauthorized users in shared threads + # inject messages into a session they don't own. from gateway.run import _AGENT_PENDING_SENTINEL if not self._is_user_authorized(event.source): logger.warning( "Dropping message from unauthorized user in active session: " - "user=%s (%s), platform=%s, session=%s", - event.source.user_id, - event.source.user_name, - event.source.platform.value if event.source.platform else "unknown", - session_key, + "user=%s (%s), platform=%s, session=%s", event.source.user_id, event.source.user_name, + event.source.platform.value if event.source.platform else "unknown", session_key, ) return True # handled (silently dropped); do not fall through effective_mode = self._effective_busy_input_mode(event.source) - - # --- Draining case (gateway restarting/stopping) --- - if self._draining: + if self._draining: # gateway restarting/stopping await self._send_busy_drain_notice(event, session_key, effective_mode) return True - if await self._route_plaintext_approval_while_busy(event, session_key): return True - - # Normal busy case (agent actively running a task) adapter = self._adapter_for_source(event.source) if not adapter: return False # let default path handle it - - # Internal synthetic events (async-delegation / background-process completions) must never - # interrupt/steer: treated as user TEXT while busy, interrupt mode would abort the active turn; - # a completion surfaces as a NEW turn only when idle. Plugin events carry untrusted payload - # text, so queue them through the gateway FIFO (security metadata kept apart). - if getattr(event, "internal", False) and not event.allow_gateway_control: + # Internal synthetic events (delegation / background completions) must never interrupt or + # steer; they surface as a NEW turn when idle. Plugin events carry untrusted payload text, so + # queue them through the FIFO (security metadata kept apart). + if getattr(event, "internal", False): + if event.allow_gateway_control: + return False self._queue_or_replace_pending_event(session_key, event) return True - if getattr(event, "internal", False): - return False - - _busy_state = self._peek_session_state(session_key) - running_agent = _busy_state.turn.agent if _busy_state else None - - busy_text_mode = self._effective_busy_text_mode(event.source) if ( event.message_type == MessageType.TEXT - and busy_text_mode == "queue" + and self._effective_busy_text_mode(event.source) == "queue" and effective_mode != "steer" ): return False + _busy_state = self._peek_session_state(session_key) + running_agent = _busy_state.turn.agent if _busy_state else None _steer = await self._resolve_busy_steer_or_redirect(event, session_key, effective_mode, running_agent) - effective_mode = _steer.effective_mode - demoted_for_subagents = _steer.demoted_for_subagents - demoted_for_compression = _steer.demoted_for_compression - steered = _steer.steered - redirected = _steer.redirected - - # Queue as the next turn after the current run ends. Skip after a successful steer — the text - # is already in the run and must NOT replay. Use the FIFO helper, not raw - # merge_pending_message_event (merge_text=True newline-joins consecutive TEXT follow-ups into - # ONE turn); FIFO gives each text its own turn while keeping photo-burst / album merge for media. - if not steered and not redirected: + effective_mode, redirected = _steer.effective_mode, _steer.redirected + # Queue as the next turn — skipped after a successful steer/redirect (the text is already in + # the run and must NOT replay). FIFO gives each text its own turn (raw merge would join them). + if not _steer.steered and not redirected: self._queue_or_replace_pending_event(session_key, event) - is_queue_mode = effective_mode == "queue" is_steer_mode = effective_mode == "steer" is_redirect_mode = effective_mode == "interrupt" and redirected - - # Interrupt mode: abort in-flight tool calls; the agent loop exits at its next check point. if ( - effective_mode == "interrupt" - and not redirected - and running_agent - and running_agent is not _AGENT_PENDING_SENTINEL + effective_mode == "interrupt" and not redirected + and running_agent and running_agent is not _AGENT_PENDING_SENTINEL ): await self._interrupt_running_agent_for_busy_event(event, adapter, running_agent) - # Disabled ack: skip sending, still process input. Checked before debounce so we never stamp a - # "last ack" timestamp for an ack that was not delivered. - busy_ack_enabled = os.environ.get("HERMES_GATEWAY_BUSY_ACK_ENABLED", "true").lower() == "true" - if not busy_ack_enabled: + # Disabled ack: still process input. Checked before debounce so an undelivered ack never + # stamps the "last ack" timestamp. + if os.environ.get("HERMES_GATEWAY_BUSY_ACK_ENABLED", "true").lower() != "true": logger.debug("Busy ack suppressed for session %s", session_key) return True # input still processed, just no ack sent - # Debounce before the config-heavy display lookup: rapid follow-ups are still processed but - # shouldn't cost a config read just to learn no ack will be sent. - _BUSY_ACK_COOLDOWN = 30 + # Debounce (30s) before the config-heavy display lookup. now = time.time() - last_ack = _busy_state.turn.busy_ack_ts if _busy_state else 0 - if now - last_ack < _BUSY_ACK_COOLDOWN: + if now - (_busy_state.turn.busy_ack_ts if _busy_state else 0) < 30: return True # interrupt sent (if not queue), ack already delivered recently if is_steer_mode and not self._busy_steer_ack_enabled(event, session_key): @@ -852,44 +639,42 @@ class GatewayBusySessionMixin: self._session_state(session_key).turn.busy_ack_ts = now message = self._compose_busy_ack_message( - event, - now, - _busy_state, - running_agent, - is_steer_mode=is_steer_mode, - is_queue_mode=is_queue_mode, - is_redirect_mode=is_redirect_mode, - demoted_for_subagents=demoted_for_subagents, - demoted_for_compression=demoted_for_compression, + event, now, _busy_state, running_agent, is_steer_mode=is_steer_mode, + is_queue_mode=is_queue_mode, is_redirect_mode=is_redirect_mode, + demoted_for_subagents=_steer.demoted_for_subagents, + demoted_for_compression=_steer.demoted_for_compression, ) await self._send_busy_ack_reply(event, adapter, message) return True + # Slash name → handler method is ``_handle__command`` (``-`` → ``_``) except these. + _COMMAND_HANDLER_ALIASES = {"bg": "_handle_background_command", "sethome": "_handle_set_home_command"} + # Ordinary slash handlers shared by idle and busy dispatch. + _PLAIN_COMMANDS = ( + "status", "context", "restart", "approve", "deny", "pause", "agents", "bg", "btw", + "kanban", "subgoal", "heartbeat", "busy", "yolo", "verbose", "footer", "help", + "commands", "profile", "update", "version", + ) + # Dispatched only on the idle path (busy dispatch has its own allowlist). + _IDLE_COMMANDS = ( + "topic", "whoami", "platform", "stop", "reasoning", "memory", "skills", "fast", + "approvals", "model", "codex-runtime", "personality", "suggestions", "save", "retry", + "sethome", "compress", "usage", "topup", "insights", "reload-mcp", "reload-skills", + "bundles", "debug", "title", "resume", "sessions", "branch", "rollback", "diff", "goal", + "loop", "refine", "review", "voice", + ) + + def _command_handler_table(self, names) -> Dict[str, Any]: + return { + name: getattr( + self, self._COMMAND_HANDLER_ALIASES.get(name, f"_handle_{name.replace('-', '_')}_command"), + ) + for name in names + } + def _gateway_plain_command_handlers(self): """Return ordinary slash handlers shared by idle and busy dispatch.""" - return { - "status": self._handle_status_command, - "context": self._handle_context_command, - "restart": self._handle_restart_command, - "approve": self._handle_approve_command, - "deny": self._handle_deny_command, - "pause": self._handle_pause_command, - "agents": self._handle_agents_command, - "bg": self._handle_background_command, - "btw": self._handle_btw_command, - "kanban": self._handle_kanban_command, - "subgoal": self._handle_subgoal_command, - "heartbeat": self._handle_heartbeat_command, - "busy": self._handle_busy_command, - "yolo": self._handle_yolo_command, - "verbose": self._handle_verbose_command, - "footer": self._handle_footer_command, - "help": self._handle_help_command, - "commands": self._handle_commands_command, - "profile": self._handle_profile_command, - "update": self._handle_update_command, - "version": self._handle_version_command, - } + return self._command_handler_table(self._PLAIN_COMMANDS) async def _send_command_ack(self, source, text: str, label: str) -> None: """Best-effort acknowledgment for a slash command that falls through to agent processing.""" @@ -903,78 +688,30 @@ class GatewayBusySessionMixin: logger.debug("%s ack send failed", label, exc_info=True) def _gateway_idle_command_handlers(self): - """Slash handlers dispatched only when no agent is running for the session (idle path). + """Slash handlers dispatched only on the idle path (busy dispatch has its own allowlist).""" + return self._command_handler_table(self._IDLE_COMMANDS) - Busy dispatch keeps its own explicit allowlist (``_dispatch_busy_slash_command``).""" - return { - "topic": self._handle_topic_command, - "whoami": self._handle_whoami_command, - "platform": self._handle_platform_command, - "stop": self._handle_stop_command, - "reasoning": self._handle_reasoning_command, - "memory": self._handle_memory_command, - "skills": self._handle_skills_command, - "fast": self._handle_fast_command, - "approvals": self._handle_approvals_command, - "model": self._handle_model_command, - "codex-runtime": self._handle_codex_runtime_command, - "personality": self._handle_personality_command, - "suggestions": self._handle_suggestions_command, - "save": self._handle_save_command, - "retry": self._handle_retry_command, - "sethome": self._handle_set_home_command, - "compress": self._handle_compress_command, - "usage": self._handle_usage_command, - "topup": self._handle_topup_command, - "insights": self._handle_insights_command, - "reload-mcp": self._handle_reload_mcp_command, - "reload-skills": self._handle_reload_skills_command, - "bundles": self._handle_bundles_command, - "debug": self._handle_debug_command, - "title": self._handle_title_command, - "resume": self._handle_resume_command, - "sessions": self._handle_sessions_command, - "branch": self._handle_branch_command, - "rollback": self._handle_rollback_command, - "diff": self._handle_diff_command, - "goal": self._handle_goal_command, - "loop": self._handle_loop_command, - "refine": self._handle_refine_command, - "review": self._handle_review_command, - "voice": self._handle_voice_command, - } + # busy_handler key (hermes_cli/commands.py CommandDef) → mid-run variant ``_busy__command``. + _BUSY_SPECIAL_HANDLERS: Dict[str, str] = { + k: f"_busy_{k}_command" for k in ("start", "stop", "new", "queue", "steer", "egress", "goal", "loop") + } - async def _dispatch_busy_slash_command( - self, event: MessageEvent, cmd_def, quick_key: str, source, - ): + async def _dispatch_busy_slash_command(self, event: MessageEvent, cmd_def, quick_key: str, source): """Dispatch a recognized slash command while an agent is running. - Order: ``busy_handler`` (special mid-run variant) → ``busy_policy == "dispatch"`` (normal - handler) → catch-all busy-reject text. Rejecting beats falling through to interrupt + - discard: Discord-registered slash commands would interrupt the agent AND be discarded by - the slash-command safety net, producing a zero-char response. + Order: ``busy_handler`` (mid-run variant) → ``busy_policy == "dispatch"`` (normal handler) + → catch-all reject text. Rejecting beats interrupt + discard (a zero-char response). """ name = cmd_def.name policy = getattr(cmd_def, "busy_policy", "reject") handler_key = getattr(cmd_def, "busy_handler", None) - if handler_key: - special = { - "start": self._busy_start_command, - "stop": self._busy_stop_command, - "new": self._busy_new_command, - "queue": self._busy_queue_command, - "steer": self._busy_steer_command, - "egress": self._busy_egress_command, - "goal": self._busy_goal_command, - "loop": self._busy_loop_command, - }.get(handler_key) + special = self._BUSY_SPECIAL_HANDLERS.get(handler_key) if special is not None: - return await special(event, quick_key, source) + return await getattr(self, special)(event, quick_key, source) reject_text = self._BUSY_REJECT_TEXT.get(handler_key) if reject_text is not None: return reject_text - if policy in ("dispatch", "interrupt_then_dispatch"): plain = self._gateway_plain_command_handlers().get(name) if plain is not None: @@ -984,21 +721,15 @@ class GatewayBusySessionMixin: "falling back to busy-reject", policy, name, ) - # Catch-all: any other recognized slash command hit the running-agent guard — reject - # gracefully rather than falling through to interrupt + discard. return ( f"⏳ Agent is running — `/{name}` can't run " f"mid-turn. Wait for the current response or `/stop` first." ) async def _handle_pause_command(self, event: MessageEvent): - """`/pause [reason]` engages the global emergency stop; `/pause off` (resume/stop) lifts it. - - In-band resume path for messaging-only operators — the estop gate lets recognized slash - commands through while paused so a user without host-shell access is never locked out. - """ + """`/pause [reason]` engages the global emergency stop; `/pause off` lifts it (the estop gate + lets slash commands through while paused so messaging-only operators are never locked out).""" from agent import estop - args = (event.get_command_args() or "").strip() if args.lower() in {"off", "resume", "stop", "disengage"}: if estop.disengage(): @@ -1006,12 +737,8 @@ class GatewayBusySessionMixin: return "Hermes wasn't paused." state = estop.get_state() if state is not None and not args: - reason = state.get("reason") - suffix = f" (reason: {reason})" if reason else "" - return ( - f"⏸️ Hermes is already paused{suffix}. " - "Use `/pause off` to resume." - ) + suffix = f" (reason: {state.get('reason')})" if state.get("reason") else "" + return f"⏸️ Hermes is already paused{suffix}. Use `/pause off` to resume." estop.engage(reason=args or None) suffix = f" (reason: {args})" if args else "" return ( @@ -1020,167 +747,116 @@ class GatewayBusySessionMixin: ) async def _busy_start_command(self, event: MessageEvent, quick_key: str, source): - # Telegram sends /start for bot launches/deep-links — a platform ping, not a user command: - # no help dump, no agent interrupt, no queued text. + # Telegram's /start is a platform ping (bot launch/deep-link), not a user command. logger.info("Ignoring /start platform ping for active session %s", quick_key) return "" async def _busy_egress_command(self, event: MessageEvent, quick_key: str, source): from hermes_cli.proxy_cli import format_status_text - return format_status_text() async def _busy_stop_command(self, event: MessageEvent, quick_key: str, source): - # /stop must hard-kill the session when an agent is running. A soft interrupt - # (agent.interrupt()) doesn't help when the agent is truly hung — the executor thread is - # blocked and never checks _interrupt_requested. + # Hard-kill: a soft interrupt can't reach a truly hung executor thread. from gateway.run import _INTERRUPT_REASON_STOP await self._interrupt_and_clear_session( - quick_key, - source, - interrupt_reason=_INTERRUPT_REASON_STOP, - invalidation_reason="stop_command", + quick_key, source, interrupt_reason=_INTERRUPT_REASON_STOP, invalidation_reason="stop_command", ) logger.info("STOP for session %s — agent interrupted, session lock released", quick_key) return EphemeralReply(t("gateway.stop.stopped")) async def _busy_new_command(self, event: MessageEvent, quick_key: str, source): - # /reset and /new must bypass the running-agent guard so they actually dispatch as commands - # instead of being queued as user text (which would be fed back to the agent with the same - # broken history — #2170). Clear any pending messages so the old text doesn't replay + # /reset and /new bypass the running-agent guard (else they'd queue as user text and replay + # into the same broken history); clear pending messages so the old text doesn't replay. from gateway.run import _INTERRUPT_REASON_RESET await self._interrupt_and_clear_session( - quick_key, - source, - interrupt_reason=_INTERRUPT_REASON_RESET, - invalidation_reason="new_command", + quick_key, source, interrupt_reason=_INTERRUPT_REASON_RESET, invalidation_reason="new_command", ) - # Clean up the running agent entry so the reset handler - # doesn't think an agent is still active. return await self._handle_reset_command(event) async def _busy_queue_command(self, event: MessageEvent, quick_key: str, source): - # /queue — queue without interrupting. Each /queue is its own full agent turn, run - # FIFO after the current run (and earlier /queue items) finish; messages are NOT merged. + # Each /queue is its own full agent turn, run FIFO after the current run; never merged. queued_text = event.get_command_args().strip() - # Preserve media/reply payloads: a /queue carrying a photo, document, or reply context is - # valid even with no prompt text (e.g. "/queue" as the caption of an image). Dropping these - # fields silently lost the attachment when the queued turn ran. + # A /queue carrying media or reply context is valid with no prompt text (image caption). has_media = bool(getattr(event, "media_urls", None)) if not queued_text and not has_media: return "Usage: /queue " adapter = self._adapter_for_source(source) if adapter: - queued_event = MessageEvent( - text=queued_text, - message_type=event.message_type if has_media else MessageType.TEXT, - source=event.source, - raw_message=event.raw_message, - message_id=event.message_id, + self._enqueue_fifo(quick_key, MessageEvent( + text=queued_text, message_type=event.message_type if has_media else MessageType.TEXT, + source=event.source, raw_message=event.raw_message, message_id=event.message_id, media_urls=list(getattr(event, "media_urls", []) or []), media_types=list(getattr(event, "media_types", []) or []), media_text_inlined=list(getattr(event, "media_text_inlined", []) or []), - reply_to_message_id=event.reply_to_message_id, - reply_to_text=event.reply_to_text, + reply_to_message_id=event.reply_to_message_id, reply_to_text=event.reply_to_text, reply_to_author_id=event.reply_to_author_id, reply_to_author_name=event.reply_to_author_name, - reply_to_is_own_message=event.reply_to_is_own_message, - auto_skill=event.auto_skill, - channel_prompt=event.channel_prompt, - channel_context=event.channel_context, - internal=event.internal, - timestamp=event.timestamp, - ) - self._enqueue_fifo(quick_key, queued_event, adapter) - depth = self._queue_depth(quick_key, adapter=self._adapter_for_source(source)) - if depth <= 1: - return "Queued for the next turn." - return f"Queued for the next turn. ({depth} queued)" + reply_to_is_own_message=event.reply_to_is_own_message, auto_skill=event.auto_skill, + channel_prompt=event.channel_prompt, channel_context=event.channel_context, + internal=event.internal, timestamp=event.timestamp, + ), adapter) + depth = self._queue_depth(quick_key, adapter=adapter) + return "Queued for the next turn." + (f" ({depth} queued)" if depth > 1 else "") async def _busy_steer_command(self, event: MessageEvent, quick_key: str, source): - # /steer — inject mid-run after the next tool call. Unlike /queue (turn boundary), - # /steer lands BETWEEN tool-call iterations inside the same agent run, by appending to the - # last tool result's content. No interrupt, no new user turn, no role-alternation violation. + # /steer lands BETWEEN tool-call iterations of the same run (appended to the last tool + # result) — no interrupt, no new user turn, no role-alternation violation. from gateway.run import _AGENT_PENDING_SENTINEL steer_text = event.get_command_args().strip() if not steer_text: return "Usage: /steer " _steer_state = self._peek_session_state(quick_key) running_agent = _steer_state.turn.agent if _steer_state else None - if running_agent is _AGENT_PENDING_SENTINEL: - # Agent hasn't started yet — queue as turn-boundary fallback. + + def _queue_fallback(reply: str) -> str: + # Turn-boundary fallback: queue the steer text as its own follow-up turn. adapter = self._adapter_for_source(source) if adapter: - queued_event = MessageEvent( - text=steer_text, - message_type=MessageType.TEXT, - source=event.source, - message_id=event.message_id, - channel_prompt=event.channel_prompt, + self._enqueue_fifo(quick_key, MessageEvent( + text=steer_text, message_type=MessageType.TEXT, source=event.source, + message_id=event.message_id, channel_prompt=event.channel_prompt, channel_context=event.channel_context, - ) - self._enqueue_fifo(quick_key, queued_event, adapter) - return "Agent still starting — /steer queued for the next turn." - if running_agent and hasattr(running_agent, "steer"): - try: - accepted = running_agent.steer(steer_text) - except Exception as exc: - logger.warning("Steer failed for session %s: %s", quick_key, exc) - return f"⚠️ Steer failed: {exc}" - if accepted: - preview = steer_text[:60] + ("..." if len(steer_text) > 60 else "") - return f"⏩ Steer queued — arrives after the next tool call: '{preview}'" + ), adapter) + return reply + + if running_agent is _AGENT_PENDING_SENTINEL: + return _queue_fallback("Agent still starting — /steer queued for the next turn.") + if not running_agent or not hasattr(running_agent, "steer"): + return _queue_fallback("No active agent — /steer queued for the next turn.") + try: + accepted = running_agent.steer(steer_text) + except Exception as exc: + logger.warning("Steer failed for session %s: %s", quick_key, exc) + return f"⚠️ Steer failed: {exc}" + if not accepted: return "Steer rejected (empty payload)." - # Running agent is missing or lacks steer() — fall back to queue. - adapter = self._adapter_for_source(source) - if adapter: - queued_event = MessageEvent( - text=steer_text, - message_type=MessageType.TEXT, - source=event.source, - message_id=event.message_id, - channel_prompt=event.channel_prompt, - channel_context=event.channel_context, - ) - self._enqueue_fifo(quick_key, queued_event, adapter) - return "No active agent — /steer queued for the next turn." + preview = steer_text[:60] + ("..." if len(steer_text) > 60 else "") + return f"⏩ Steer queued — arrives after the next tool call: '{preview}'" async def _busy_goal_command(self, event: MessageEvent, quick_key: str, source): - # /goal is safe mid-run for status/pause/clear/wait (inspection and control-plane only — - # doesn't interrupt the running turn). Setting new goal text mid-run is rejected like - # /model so we don't race a second continuation prompt against the current turn. + # Control verbs are safe mid-run (state only); setting new goal text is rejected so we don't + # race a second continuation against the current turn. wait/gate take an argument. _goal_arg = (event.get_command_args() or "").strip().lower() _goal_verb = _goal_arg.split(None, 1)[0] if _goal_arg else "" - # Exact-match control verbs, plus the wait/unwait barrier verbs (take a pid) and the gate - # management verb (gates run at turn boundary, so editing the gate list mid-run is safe). - _is_control = ( + if ( not _goal_arg or _goal_arg in {"status", "pause", "resume", "clear", "stop", "done", "unwait"} or _goal_verb in {"wait", "gate"} - ) - if _is_control: + ): return await self._handle_goal_command(event) return "Agent is running — use /goal status / pause / clear / wait mid-run, or /stop before setting a new goal." async def _busy_loop_command(self, event: MessageEvent, quick_key: str, source): - # /loop mirrors /goal: control verbs are safe mid-run (state only — read at the next idle - # boundary); setting a new loop mid-run is rejected so we don't race the current turn. + # Mirrors /goal: control verbs are safe mid-run; a new loop is rejected. _loop_arg = (event.get_command_args() or "").strip().lower() if not _loop_arg or _loop_arg in {"status", "pause", "resume", "stop", "clear", "cancel", "help", "--help", "-h"}: return await self._handle_loop_command(event) return "Agent is running — use /loop status / pause / stop mid-run, or /stop before setting a new loop." - def _check_slash_access( - self, source: SessionSource, canonical_cmd: str - ) -> Optional[str]: - """Return a denial message if ``source`` cannot run ``canonical_cmd``, else None. - - Used by the cold and running-agent dispatch paths in ``_handle_message`` so admin/user gating - can't be bypassed by an in-flight agent. Backward-compat: without ``allow_admin_from`` for the - scope, ``policy_for_source`` returns ``enabled=False`` and this always returns None. - """ + def _check_slash_access(self, source: SessionSource, canonical_cmd: str) -> Optional[str]: + """Denial message if ``source`` cannot run ``canonical_cmd``, else None (both dispatch paths + use it so an in-flight agent can't bypass admin gating; no ``allow_admin_from`` → None).""" from gateway.slash_access import policy_for_source as _policy_for_source - if not canonical_cmd: return None policy = _policy_for_source(self.config, source) @@ -1188,35 +864,25 @@ class GatewayBusySessionMixin: return None logger.info( "Slash command /%s denied for %s:%s (not admin, not in user_allowed_commands)", - canonical_cmd, - source.platform.value if source.platform else "?", - source.user_id, + canonical_cmd, source.platform.value if source.platform else "?", source.user_id, ) allowed_preview = sorted(policy.user_allowed_commands) if allowed_preview: suffix = ( - "You can run: " - + ", ".join(f"/{c}" for c in allowed_preview[:12]) - + ("…" if len(allowed_preview) > 12 else "") - + ". Use /whoami for the full list." + "You can run: " + ", ".join(f"/{c}" for c in allowed_preview[:12]) + + ("…" if len(allowed_preview) > 12 else "") + ". Use /whoami for the full list." ) else: suffix = ( - "No slash commands are enabled for non-admins on this " - "platform. Ask an admin to add you to allow_admin_from " - "or to set user_allowed_commands." + "No slash commands are enabled for non-admins on this platform. Ask an admin to " + "add you to allow_admin_from or to set user_allowed_commands." ) return f"⛔ /{canonical_cmd} is admin-only here. {suffix}" def _sibling_thread_run_keys(self, source: SessionSource, own_key: str) -> list: - """Find running-agent keys for OTHER participants in the same thread. - - In per-user thread mode each participant gets an isolated key - (``...:{thread_id}:{user_id}``), so another user's run is invisible to the caller's own - ``/stop``. Returns keys of *actually running* agents (not the pending sentinel, not the - caller's own) sharing the caller's ``{chat_id}:{thread_id}`` prefix; empty when not in a - thread or no sibling runs exist. Callers must still gate on authorization. - """ + """Running-agent keys of OTHER participants in the same thread (per-user thread mode keys + are ``...:{thread_id}:{user_id}``, so another user's run is invisible to the caller's own + ``/stop``). Excludes the pending sentinel and ``own_key``; callers still gate on authz.""" from gateway.run import _AGENT_PENDING_SENTINEL thread_id = getattr(source, "thread_id", None) chat_id = getattr(source, "chat_id", None) @@ -1224,57 +890,39 @@ class GatewayBusySessionMixin: return [] platform = source.platform.value chat_type = getattr(source, "chat_type", None) or "" - # Prefix that every per-user key in this thread shares, up to and including the thread_id - # segment. Match the exact key or prefix + ":" (a further user_id segment) so an unrelated - # thread whose id merely starts with this one is not matched. - prefix = ":".join( - ["agent:main", platform, chat_type, str(chat_id), str(thread_id)] - ) - matches = [] - for key, agent in self._running_agent_items(): - if key == own_key: - continue - if agent is _AGENT_PENDING_SENTINEL or not agent: - continue - if key == prefix or key.startswith(prefix + ":"): - matches.append(key) - return matches + # Match the exact key or prefix + ":" so a thread id that merely starts with this one + # is not matched. + prefix = ":".join(["agent:main", platform, chat_type, str(chat_id), str(thread_id)]) + return [ + key + for key, agent in self._running_agent_items() + if key != own_key + and agent is not _AGENT_PENDING_SENTINEL and agent + and (key == prefix or key.startswith(prefix + ":")) + ] def _is_stale_restart_redelivery(self, event: MessageEvent) -> bool: - """Return True if this /restart is a Telegram re-delivery we already handled. + """True if this /restart is a Telegram re-delivery we already handled. - The previous gateway wrote ``.restart_last_processed.json`` with the triggering platform - + update_id when it processed the /restart. A /restart on the same platform with - update_id <= that value is a redelivery when this process booted from that restart; - otherwise the marker must still be recent (< 5 minutes). Telegram only (the only platform - with a numeric cross-session update ordering); other platforms return False. + The previous gateway wrote ``.restart_last_processed.json`` (platform + update_id). A + /restart with update_id <= that value is a redelivery when this process booted from that + restart; otherwise the marker must be < 5 minutes old. Telegram only (numeric ordering). """ from gateway.run import _hermes_home - if event is None or event.source is None: + if event is None or event.source is None or event.platform_update_id is None: return False - if event.platform_update_id is None: - return False - if event.source.platform is None: - return False - # Only Telegram populates platform_update_id currently; be explicit - # so future platforms aren't accidentally gated by this check. try: - platform_value = event.source.platform.value + if event.source.platform.value != "telegram": + return False except Exception: return False - if platform_value != "telegram": - return False try: marker_path = _hermes_home / ".restart_last_processed.json" if not marker_path.exists(): - # Belt-and-suspenders for a missing dedup marker (cleaned up, or the previous write - # failed): without it the update_id comparison can't run and a redelivered /restart - # would re-restart the gateway forever. Suppress ONLY when a restart cycle is - # independently confirmed: this process booted from a chat-originated /restart - # (_booted_from_restart) AND is within a short post-boot window; a genuine first - # /restart on a fresh boot is never swallowed (flag stays False). Consume the flag - # one-shot so a later legitimate /restart in the same session is honored. + # Missing marker: a redelivered /restart would otherwise re-restart forever. Suppress + # ONLY when this process booted from a chat /restart AND is within a short post-boot + # window; consume the flag one-shot so a later legitimate /restart is honored. if ( getattr(self, "_booted_from_restart", False) and time.time() - getattr(self, "_startup_time", 0.0) < 60 @@ -1286,149 +934,77 @@ class GatewayBusySessionMixin: except Exception: return False - if data.get("platform") != platform_value: - return False recorded_uid = data.get("update_id") - if not isinstance(recorded_uid, int): - return False - if event.platform_update_id > recorded_uid: + if ( + data.get("platform") != "telegram" + or not isinstance(recorded_uid, int) + or event.platform_update_id > recorded_uid + ): return False - # A service-managed restart can legitimately take longer than the marker's normal five- - # minute trust window while adapters, cron, and in-flight deliveries drain. Consume the boot - # signal one-shot so a later genuine command is evaluated normally. + # A service-managed restart can outlast the 5-minute trust window; consume the boot + # signal one-shot. if getattr(self, "_booted_from_restart", False): self._booted_from_restart = False return True - # Staleness guard: ignore markers older than 5 minutes so a legitimately old one (e.g. crash - # recovery where notify never fired) doesn't swallow a fresh /restart. + # Staleness guard: an old marker (crash recovery) must not swallow a fresh /restart. requested_at = data.get("requested_at") - if isinstance(requested_at, (int, float)): - if time.time() - requested_at > 300: - return False - return True + return not (isinstance(requested_at, (int, float)) and time.time() - requested_at > 300) async def _handle_suggestions_command(self, event: MessageEvent) -> str: - """Handle /suggestions in the gateway. - - Delegates to the shared handler so CLI and gateway never drift. The origin is built from - the event source so an accepted suggestion's job delivers back to this chat/thread. - """ + """/suggestions via the shared handler (origin = event source so jobs deliver back here).""" from gateway.run import _command_origin_for_source - args = (event.get_command_args() or "").strip() - origin = _command_origin_for_source(event.source) try: from hermes_cli.suggestions_cmd import handle_suggestions_command - - return handle_suggestions_command(args, origin=origin, surface="gateway") + return handle_suggestions_command( + (event.get_command_args() or "").strip(), + origin=_command_origin_for_source(event.source), surface="gateway", + ) except Exception as e: logger.debug("suggestions command failed: %s", e) return f"Suggestions command failed: {e}" async def _handle_blueprint_command(self, event: MessageEvent): - """Handle /blueprint in the gateway. - - Delegates to the shared handler so CLI, TUI, and gateway never drift. Origin is built - from the event source so a directly created blueprint job delivers back to this chat. - """ + """/blueprint via the shared handler (origin = event source so jobs deliver back here).""" from gateway.run import _command_origin_for_source - args = (event.get_command_args() or "").strip() - origin = _command_origin_for_source(event.source) try: from hermes_cli.blueprint_cmd import handle_blueprint_command - - return handle_blueprint_command(args, origin=origin, surface="gateway") + return handle_blueprint_command( + (event.get_command_args() or "").strip(), + origin=_command_origin_for_source(event.source), surface="gateway", + ) except Exception as e: logger.debug("blueprint command failed: %s", e) from hermes_cli.blueprint_cmd import BlueprintCommandResult - return BlueprintCommandResult(f"Cron blueprint command failed: {e}") async def _maybe_confirm_destructive_slash( - self, - *, - event: MessageEvent, - command: str, - title: str, - detail: str, - execute, + self, *, event: MessageEvent, command: str, title: str, detail: str, execute ) -> Union[str, "EphemeralReply", None]: """Gate a destructive session slash command (/new, /reset, /undo). - ``execute`` is an async ``execute() -> str | EphemeralReply`` performing the action. It - runs immediately if ``approvals.destructive_slash_confirm`` is off; otherwise this routes - through ``_request_slash_confirm`` (native buttons or text fallback): ``once`` runs it, - ``always`` persists ``destructive_slash_confirm: false`` then runs it, ``cancel`` returns - a "cancelled" message without running it. + ``execute()`` (async → str | EphemeralReply) runs immediately when + ``approvals.destructive_slash_confirm`` is off; otherwise via ``_request_slash_confirm``: + ``once`` runs it, ``always`` persists the opt-out then runs it, ``cancel`` skips it. """ - # Gate check. confirm_required = True try: - cfg = self._read_user_config() - approvals = cfg.get("approvals") if isinstance(cfg, dict) else None + approvals = self._read_user_config().get("approvals") if isinstance(approvals, dict): confirm_required = bool(approvals.get("destructive_slash_confirm", True)) except Exception: pass - if not confirm_required: return await execute() session_key = self._session_key_for_source(event.source) async def _on_confirm(choice: str): - if choice == "cancel": - return f"🟡 /{command} cancelled. Conversation unchanged." - persisted = False - if choice == "always": - try: - from cli import save_config_value - # save_config_value swallows its own errors and reports the - # outcome in the return value, so the try block alone says - # nothing about whether the write landed. - persisted = bool( - save_config_value("approvals.destructive_slash_confirm", False) - ) - if persisted: - logger.info( - "User opted out of destructive slash confirm (session=%s)", - session_key, - ) - else: - logger.warning( - "Could not persist destructive_slash_confirm=false " - "(session=%s); config.yaml is not writable", - session_key, - ) - except Exception as exc: - logger.warning( - "Failed to persist destructive_slash_confirm=false: %s", exc, - ) - result = await execute() - if choice == "always": - if persisted: - note = ( - "\n\nℹ️ Future /clear, /new, /reset, and /undo will run " - "without confirmation. Re-enable via " - "`approvals.destructive_slash_confirm: true` in config.yaml." - ) - else: - # The user did approve this run, so the action still goes ahead, but the - # preference did not stick and the prompt will be back next time. Say so rather - # than promising an opt-out that was never written. - note = ( - "\n\n⚠️ Could not save that preference (config.yaml is not " - "writable), so /clear, /new, /reset, and /undo will ask " - "again next time. To silence it permanently, set " - "`approvals.destructive_slash_confirm: false` in config.yaml." - ) - if isinstance(result, str): - return result + note - # EphemeralReply or other: leave untouched, since the note would - # mangle structured replies. - return result - return result + # Via the class, not ``self``: tests drive this gate on a bare SimpleNamespace runner. + return await GatewayBusySessionMixin._run_confirmed_destructive_slash( + choice, command, execute, session_key + ) _p = self._typed_command_prefix_for(event.source.platform) prompt_message = ( @@ -1441,83 +1017,93 @@ class GatewayBusySessionMixin: f"_Text fallback: reply `{_p}approve`, `{_p}always`, or `{_p}cancel`._" ) return await self._request_slash_confirm( - event=event, - command=command, - title=title, - message=prompt_message, - handler=_on_confirm, + event=event, command=command, title=title, message=prompt_message, handler=_on_confirm ) + _DESTRUCTIVE_OPTOUT_NOTE = { + True: ( + "\n\nℹ️ Future /clear, /new, /reset, and /undo will run " + "without confirmation. Re-enable via " + "`approvals.destructive_slash_confirm: true` in config.yaml." + ), + # The user did approve this run, so the action still goes ahead, but the preference did + # not stick and the prompt will be back next time. Say so rather than promising an + # opt-out that was never written. + False: ( + "\n\n⚠️ Could not save that preference (config.yaml is not " + "writable), so /clear, /new, /reset, and /undo will ask " + "again next time. To silence it permanently, set " + "`approvals.destructive_slash_confirm: false` in config.yaml." + ), + } + + @staticmethod + async def _run_confirmed_destructive_slash(choice: str, command: str, execute, session_key: str): + """Confirm-callback body: ``cancel`` → message; ``always`` persists the opt-out, then runs.""" + if choice == "cancel": + return f"🟡 /{command} cancelled. Conversation unchanged." + persisted = False + if choice == "always": + try: + from cli import save_config_value + # save_config_value swallows its own errors and reports the outcome in the return + # value, so the try block alone says nothing about whether the write landed. + persisted = bool(save_config_value("approvals.destructive_slash_confirm", False)) + if persisted: + logger.info("User opted out of destructive slash confirm (session=%s)", session_key) + else: + logger.warning( + "Could not persist destructive_slash_confirm=false " + "(session=%s); config.yaml is not writable", session_key, + ) + except Exception as exc: + logger.warning("Failed to persist destructive_slash_confirm=false: %s", exc) + result = await execute() + # Only plain-string results get the note: it would mangle an EphemeralReply. + if choice == "always" and isinstance(result, str): + return result + GatewayBusySessionMixin._DESTRUCTIVE_OPTOUT_NOTE[persisted] + return result + async def _request_slash_confirm( - self, - *, - event: MessageEvent, - command: str, - title: str, - message: str, - handler, + self, *, event: MessageEvent, command: str, title: str, message: str, handler ) -> Optional[str]: - """Ask the user to confirm an expensive slash command. - - ``handler(choice: str) -> str`` runs on the event loop when the user responds with - ``"once"``, ``"always"``, or ``"cancel"``; its return value is sent as a gateway message. - Returns the immediate acknowledgment: ``None`` if buttons rendered (self-explanatory), - otherwise the text-fallback message itself IS the ack. - """ + """Ask the user to confirm a slash command; ``handler(choice)`` runs on "once"/"always"/ + "cancel" and its return is sent as a message. Returns None if buttons rendered, else the + text-fallback message (which IS the ack).""" from tools import slash_confirm as _slash_confirm_mod - source = event.source session_key = self._session_key_for_source(source) - # Bare-runner test harnesses (object.__new__(GatewayRunner)) skip __init__ and lack the - # counter attribute; fall back to a local counter. Real runs always have the attribute. + # object.__new__ test runners lack the counter; fall back to a local one. counter = getattr(self, "_slash_confirm_counter", None) if counter is None: import itertools as _itertools - counter = _itertools.count(1) - self._slash_confirm_counter = counter + counter = self._slash_confirm_counter = _itertools.count(1) confirm_id = f"{next(counter)}" - # Register the pending confirm FIRST so a super-fast button click - # cannot race the send_slash_confirm return. + # Register FIRST so a fast button click cannot race the send_slash_confirm return. _slash_confirm_mod.register(session_key, confirm_id, command, handler) adapter = self._adapter_for_source(source) metadata = self._thread_metadata_for_source(source, self._reply_anchor_for_event(event)) - used_buttons = False if adapter is not None: try: button_result = await adapter.send_slash_confirm( - chat_id=source.chat_id, - title=title, - message=message, - session_key=session_key, - confirm_id=confirm_id, - metadata=metadata, + chat_id=source.chat_id, title=title, message=message, session_key=session_key, + confirm_id=confirm_id, metadata=metadata, ) if button_result and getattr(button_result, "success", False): - used_buttons = True + return None # buttons rendered — no redundant text ack except Exception as exc: - logger.debug( - "send_slash_confirm failed for %s on %s: %s", - command, source.platform, exc, - ) - - if used_buttons: - # Buttons rendered — no redundant text ack. - return None - # Text fallback — return the prompt message as the direct reply. + logger.debug("send_slash_confirm failed for %s on %s: %s", command, source.platform, exc) + # Text fallback — the prompt message itself is the direct reply. return message def _read_user_config(self) -> Dict[str, Any]: - """Read the user's raw config.yaml (cached) for gate lookups. - - Used by slash-confirm gates that must reflect on-disk state changes - (e.g. a prior "Always Approve" click) without a gateway restart. - """ + """Raw config.yaml for gate lookups that must see on-disk changes without a restart.""" try: from hermes_cli.config import load_config cfg = load_config() - return cfg if isinstance(cfg, dict) else {} except Exception: return {} + return cfg if isinstance(cfg, dict) else {} diff --git a/gateway/run_config_loaders.py b/gateway/run_config_loaders.py index bf64fe6b67..d209f9c702 100644 --- a/gateway/run_config_loaders.py +++ b/gateway/run_config_loaders.py @@ -1,4 +1,4 @@ -"""Config/env loaders for runtime knobs (busy modes, reasoning, service tier, timeouts, fallback) for GatewayRunner. +"""Config/env loaders (busy modes, reasoning, service tier, timeouts, fallback) for GatewayRunner. Split out of ``gateway/run.py``; bound onto ``GatewayRunner`` via the MRO. ``gateway.run`` internals are imported lazily inside method bodies (import cycle), @@ -7,29 +7,26 @@ so ``patch("gateway.run.X")`` keeps intercepting them at call time. from __future__ import annotations -import logging -from typing import TYPE_CHECKING import json +import logging import os import time +from contextlib import suppress +from pathlib import Path +from typing import TYPE_CHECKING, Any, Dict, List, Optional + from gateway.config import Platform from gateway.restart import ( - DEFAULT_GATEWAY_CRON_DRAIN_TIMEOUT, - DEFAULT_GATEWAY_POST_INTERRUPT_GRACE_TIMEOUT, - DEFAULT_GATEWAY_RESTART_AFTER_TURN_TIMEOUT, - DEFAULT_GATEWAY_RESTART_DRAIN_TIMEOUT, - DEFAULT_GATEWAY_SIGNAL_INTERRUPT_GRACE_TIMEOUT, - parse_cron_drain_timeout, - parse_restart_after_turn_timeout, - parse_restart_drain_timeout, + DEFAULT_GATEWAY_CRON_DRAIN_TIMEOUT, DEFAULT_GATEWAY_POST_INTERRUPT_GRACE_TIMEOUT, + DEFAULT_GATEWAY_RESTART_AFTER_TURN_TIMEOUT, DEFAULT_GATEWAY_RESTART_DRAIN_TIMEOUT, + DEFAULT_GATEWAY_SIGNAL_INTERRUPT_GRACE_TIMEOUT, parse_cron_drain_timeout, + parse_restart_after_turn_timeout, parse_restart_drain_timeout, parse_signal_interrupt_grace_timeout, ) from gateway.session import SessionSource from gateway.session_state import SERVICE_TIER_UNSET as _SERVICE_TIER_UNSET -from hermes_cli.config import cfg_get +from hermes_cli.config import cfg_get, resolve_ephemeral_system_prompt_from_config from hermes_cli.fallback_config import get_fallback_chain -from pathlib import Path -from typing import Any, Dict, List, Optional from utils import is_truthy_value if TYPE_CHECKING: # string annotations only; never imported at runtime (cycle) @@ -38,9 +35,22 @@ if TYPE_CHECKING: # string annotations only; never imported at runtime (cycle) # Log-record parity with the origin module. logger = logging.getLogger("gateway.run") +_BUSY_INPUT_MODES = {"interrupt", "queue", "steer"} + class GatewayConfigLoadersMixin: - """Config/env loaders for runtime knobs (busy modes, reasoning, service tier, timeouts, fallback) for GatewayRunner.""" + """Config/env loaders (busy modes, reasoning, service tier, timeouts, fallback) for GatewayRunner.""" + + @staticmethod + def _cfg_str(section: str, key: str) -> str: + """``

.`` from the gateway runtime config as a stripped string ("" when unset).""" + from gateway.run import _load_gateway_runtime_config + return str(cfg_get(_load_gateway_runtime_config(), section, key, default="") or "").strip() + + @classmethod + def _env_or_cfg_str(cls, env_var: str, section: str, key: str) -> str: + """Non-empty stripped env var, else :meth:`_cfg_str`.""" + return os.getenv(env_var, "").strip() or cls._cfg_str(section, key) @staticmethod def _load_prefill_messages() -> List[Dict[str, Any]]: @@ -53,9 +63,9 @@ class GatewayConfigLoadersMixin: file_path = os.getenv("HERMES_PREFILL_MESSAGES_FILE", "") if not file_path: cfg = _load_gateway_runtime_config() - file_path = str(cfg.get("prefill_messages_file", "") or "") - if not file_path: - file_path = str(cfg_get(cfg, "agent", "prefill_messages_file", default="") or "") + file_path = str( + cfg.get("prefill_messages_file", "") or cfg_get(cfg, "agent", "prefill_messages_file", default="") or "" + ) if not file_path: return [] path = Path(file_path).expanduser() @@ -77,26 +87,24 @@ class GatewayConfigLoadersMixin: @staticmethod def _load_ephemeral_system_prompt() -> str: - """Load ephemeral system prompt: HERMES_EPHEMERAL_SYSTEM_PROMPT env var first, then - ``display.personality`` / ``agent.system_prompt`` in config.yaml. - """ + """HERMES_EPHEMERAL_SYSTEM_PROMPT env first, then ``display.personality`` / ``agent.system_prompt``.""" from gateway.run import _load_gateway_runtime_config - from hermes_cli.config import resolve_ephemeral_system_prompt_from_config - prompt = os.getenv("HERMES_EPHEMERAL_SYSTEM_PROMPT", "") if prompt: return prompt - cfg = _load_gateway_runtime_config() - return resolve_ephemeral_system_prompt_from_config(cfg) + return resolve_ephemeral_system_prompt_from_config(_load_gateway_runtime_config()) + + def _channel_override(self, platform: Platform, chat_id: str, thread_id, parent_id): + """``channel_overrides`` entry for this channel/thread, or None (also when no config is bound).""" + from gateway.run import _get_channel_override + config = getattr(self, "config", None) + if not config: + return None + return _get_channel_override(config, platform, chat_id, thread_id=thread_id, parent_id=parent_id) def _resolve_model_for_channel( - self, - platform: Platform, - chat_id: str, - *, - user_config: Optional[dict] = None, - thread_id: Optional[str] = None, - parent_id: Optional[str] = None, + self, platform: Platform, chat_id: str, *, user_config: Optional[dict] = None, + thread_id: Optional[str] = None, parent_id: Optional[str] = None, ) -> str: """Resolve model for this channel: channel_overrides else global default. @@ -104,31 +112,16 @@ class GatewayConfigLoadersMixin: API server so the surfaces cannot diverge). No session tier here: session /model overrides are applied later by ``_apply_session_model_override``. """ - from gateway.run import _get_channel_override, _resolve_gateway_model + from gateway.run import _resolve_gateway_model from hermes_cli.model_switch import resolve_effective_model - - override = None - config = getattr(self, "config", None) - if config: - override = _get_channel_override( - config, - platform, - chat_id, - thread_id=thread_id, - parent_id=parent_id, - ) return resolve_effective_model( None, # session tier applied downstream (_apply_session_model_override) - override, + self._channel_override(platform, chat_id, thread_id, parent_id), _resolve_gateway_model(user_config), ) def _get_system_prompt_for_channel( - self, - platform: Platform, - chat_id: str, - *, - thread_id: Optional[str] = None, + self, platform: Platform, chat_id: str, *, thread_id: Optional[str] = None, parent_id: Optional[str] = None, ) -> str: """Ephemeral system prompt for this channel/thread. @@ -138,40 +131,26 @@ class GatewayConfigLoadersMixin: profiles get their own personality/system_prompt and ``/personality`` edits apply next turn). Legacy ``channel_prompts`` are applied separately via ``event.channel_prompt`` in ``run_sync``. """ - from gateway.run import _get_channel_override - config = getattr(self, "config", None) - if config: - override = _get_channel_override( - config, - platform, - chat_id, - thread_id=thread_id, - parent_id=parent_id, - ) - if override and override.system_prompt: - return (override.system_prompt or "").strip() + override = self._channel_override(platform, chat_id, thread_id, parent_id) + if override and override.system_prompt: + return (override.system_prompt or "").strip() return self._load_ephemeral_system_prompt() @staticmethod def _load_reasoning_config(model: str = "") -> dict | None: - """Load reasoning effort from config.yaml, respecting per-model overrides. + """Reasoning effort from config.yaml via :func:`hermes_constants.resolve_reasoning_config`. - Thin wrapper over :func:`hermes_constants.resolve_reasoning_config` (per-model override > - global ``agent.reasoning_effort``; YAML False = disabled). Empty ``model`` uses ``model.default``. + Per-model override > global ``agent.reasoning_effort``; YAML False = disabled. Empty + ``model`` uses ``model.default``. """ from gateway.run import _load_gateway_runtime_config from hermes_constants import resolve_reasoning_config - cfg = _load_gateway_runtime_config() - return resolve_reasoning_config(cfg, model) + return resolve_reasoning_config(_load_gateway_runtime_config(), model) @staticmethod def _parse_reasoning_command_args(raw_args: str) -> tuple[str, bool]: - """Parse `/reasoning` args into `(value, persist_global)`. - - Session-scoped by default; `--global` in any position persists the change to config.yaml. - """ + """Parse `/reasoning` args into `(value, persist_global)`; `--global` anywhere persists to config.""" import shlex - text = str(raw_args or "").strip().replace("—", "--") if not text: return "", False @@ -179,42 +158,26 @@ class GatewayConfigLoadersMixin: tokens = shlex.split(text) except ValueError: tokens = text.split() - - persist_global = False - value_tokens = [] - for token in tokens: - if token == "--global": - persist_global = True - else: - value_tokens.append(token) - return " ".join(value_tokens).strip().lower(), persist_global + value = " ".join(token for token in tokens if token != "--global") + return value.strip().lower(), "--global" in tokens def _resolve_session_reasoning_config( - self, - *, - source: Optional[SessionSource] = None, - session_key: Optional[str] = None, + self, *, source: Optional[SessionSource] = None, session_key: Optional[str] = None, model: str = "", ) -> dict | None: - """Resolve reasoning effort for a session, honoring session overrides. + """Session ``/reasoning --session`` > per-model ``agent.reasoning_overrides`` > global. - Priority: session ``/reasoning --session`` > per-model ``agent.reasoning_overrides`` > global - ``agent.reasoning_effort``. ``model`` must be the session's *effective* model (session - ``/model`` override included); empty uses ``model.default``. + ``model`` must be the session's *effective* model (session ``/model`` override included); + empty uses ``model.default``. """ resolved_session_key = self._resolve_session_key_or_none(source, session_key) - if resolved_session_key: _r_state = self._peek_session_state(resolved_session_key) if _r_state is not None and _r_state.conversation.reasoning_override is not None: return _r_state.conversation.reasoning_override return self._load_reasoning_config(model) - def _set_session_reasoning_override( - self, - session_key: str, - reasoning_config: Optional[dict], - ) -> None: + def _set_session_reasoning_override(self, session_key: str, reasoning_config: Optional[dict]) -> None: """Set or clear the session-scoped reasoning override.""" if not session_key: return @@ -224,39 +187,20 @@ class GatewayConfigLoadersMixin: None if reasoning_config is None else dict(reasoning_config) ) - def _resolve_session_service_tier( - self, - source=None, - session_key: Optional[str] = None, - ) -> Optional[str]: - """Resolve the effective service tier for a session. + def _resolve_session_service_tier(self, source=None, session_key: Optional[str] = None) -> Optional[str]: + """Effective service tier: a session-scoped /fast override beats the config default. - A session-scoped /fast override beats the config default; the override dict stores - "priority" or None (explicit normal), so key presence — not truthiness — decides. + The override stores "priority" or None (explicit normal), so presence — not truthiness — decides. """ resolved_session_key = self._resolve_session_key_or_none(source, session_key) - if resolved_session_key: _t_state = self._peek_session_state(resolved_session_key) - if ( - _t_state is not None - and _t_state.conversation.service_tier_override - is not _SERVICE_TIER_UNSET - ): + if _t_state is not None and _t_state.conversation.service_tier_override is not _SERVICE_TIER_UNSET: return _t_state.conversation.service_tier_override return self._load_service_tier() - def _set_session_service_tier_override( - self, - session_key: str, - service_tier, - clear: bool = False, - ) -> None: - """Set or clear the session-scoped /fast override. - - ``service_tier`` is "priority" or None (explicit normal). Pass - ``clear=True`` to remove the override entirely (fall back to config). - """ + def _set_session_service_tier_override(self, session_key: str, service_tier, clear: bool = False) -> None: + """Set ("priority" / None = explicit normal) or ``clear`` the session-scoped /fast override.""" if not session_key: return # Presence-sensitive: "priority" or None (explicit normal) both count as an override; the @@ -265,15 +209,10 @@ class GatewayConfigLoadersMixin: _SERVICE_TIER_UNSET if clear else service_tier ) - @staticmethod - def _load_service_tier() -> str | None: - """Load Priority Processing (agent.service_tier) from config.yaml: "fast"/"priority"/"on" => - "priority"; "normal"/"off" disable; None when unset/unsupported. - """ - from gateway.run import _load_gateway_runtime_config - cfg = _load_gateway_runtime_config() - raw = str(cfg_get(cfg, "agent", "service_tier", default="") or "").strip() - + @classmethod + def _load_service_tier(cls) -> str | None: + """``agent.service_tier``: fast/priority/on => "priority"; normal/off => None; None when unset/unknown.""" + raw = cls._cfg_str("agent", "service_tier") value = raw.lower() if not value or value in {"normal", "default", "standard", "off", "none"}: return None @@ -286,72 +225,38 @@ class GatewayConfigLoadersMixin: @staticmethod def _load_show_reasoning() -> bool: - """Load show_reasoning toggle from config.yaml display section.""" + """``display.show_reasoning`` toggle.""" from gateway.run import _load_gateway_runtime_config - cfg = _load_gateway_runtime_config() - return is_truthy_value( - cfg_get(cfg, "display", "show_reasoning"), - default=False, - ) + return is_truthy_value(cfg_get(_load_gateway_runtime_config(), "display", "show_reasoning"), default=False) - @staticmethod - def _load_busy_input_mode() -> str: - """Load gateway drain-time busy-input behavior from config/env.""" - from gateway.run import _load_gateway_runtime_config - mode = os.getenv("HERMES_GATEWAY_BUSY_INPUT_MODE", "").strip().lower() - if not mode: - cfg = _load_gateway_runtime_config() - mode = str(cfg_get(cfg, "display", "busy_input_mode", default="") or "").strip().lower() - if mode == "queue": - return "queue" - if mode == "steer": - return "steer" - return "interrupt" + @classmethod + def _load_busy_input_mode(cls) -> str: + """Gateway drain-time busy-input behavior from env/config (default ``interrupt``).""" + mode = cls._env_or_cfg_str("HERMES_GATEWAY_BUSY_INPUT_MODE", "display", "busy_input_mode").lower() + return mode if mode in {"queue", "steer"} else "interrupt" - @staticmethod - def _load_busy_text_mode() -> str: - """Resolve normal busy TEXT follow-up behavior. + @classmethod + def _load_busy_text_mode(cls) -> str: + """Normal busy TEXT follow-up behavior. ``busy_input_mode`` is the source of truth (default ``interrupt``); legacy ``busy_text_mode`` is honored only when explicitly set so existing queue setups keep working. """ - from gateway.run import GatewayRunner, _load_gateway_runtime_config - # Legacy explicit override wins for backward compat. - legacy = os.getenv("HERMES_GATEWAY_BUSY_TEXT_MODE", "").strip().lower() - if not legacy: - cfg = _load_gateway_runtime_config() - legacy = str(cfg_get(cfg, "display", "busy_text_mode", default="") or "").strip().lower() - if legacy == "interrupt": - return "interrupt" - if legacy == "queue": - return "queue" - # No explicit legacy knob → follow busy_input_mode. - input_mode = GatewayRunner._load_busy_input_mode() - return "queue" if input_mode == "queue" else "interrupt" + from gateway.run import GatewayRunner + legacy = cls._env_or_cfg_str("HERMES_GATEWAY_BUSY_TEXT_MODE", "display", "busy_text_mode").lower() + if legacy in {"interrupt", "queue"}: + return legacy + return "queue" if GatewayRunner._load_busy_input_mode() == "queue" else "interrupt" @staticmethod - def _busy_modes_from_config( - config: dict, - *, - fallback_input: str, - fallback_text: str, - ) -> tuple[str, str]: + def _busy_modes_from_config(config: dict, *, fallback_input: str, fallback_text: str) -> tuple[str, str]: """Resolve one profile's busy modes without consulting process env.""" - raw_input = str( - cfg_get(config, "display", "busy_input_mode", default="") or "" - ).strip().lower() - input_mode = ( - raw_input - if raw_input in {"interrupt", "queue", "steer"} - else fallback_input - ) - - raw_text = str( - cfg_get(config, "display", "busy_text_mode", default="") or "" - ).strip().lower() + raw_input = str(cfg_get(config, "display", "busy_input_mode", default="") or "").strip().lower() + input_mode = raw_input if raw_input in _BUSY_INPUT_MODES else fallback_input + raw_text = str(cfg_get(config, "display", "busy_text_mode", default="") or "").strip().lower() if raw_text in {"interrupt", "queue"}: text_mode = raw_text - elif raw_input in {"interrupt", "queue", "steer"}: + elif raw_input in _BUSY_INPUT_MODES: text_mode = "queue" if input_mode == "queue" else "interrupt" else: text_mode = fallback_text @@ -360,14 +265,11 @@ class GatewayConfigLoadersMixin: def _snapshot_profile_busy_modes(self, profile_name: str, config: dict) -> None: """Cache a routed profile's busy policy for this gateway lifetime.""" input_mode, text_mode = self._busy_modes_from_config( - config, - fallback_input=getattr(self, "_busy_input_mode", "interrupt"), + config, fallback_input=getattr(self, "_busy_input_mode", "interrupt"), fallback_text=getattr(self, "_busy_text_mode", "interrupt"), ) - input_modes = self.__dict__.setdefault("_busy_input_modes_by_profile", {}) - text_modes = self.__dict__.setdefault("_busy_text_modes_by_profile", {}) - input_modes[profile_name] = input_mode - text_modes[profile_name] = text_mode + self.__dict__.setdefault("_busy_input_modes_by_profile", {})[profile_name] = input_mode + self.__dict__.setdefault("_busy_text_modes_by_profile", {})[profile_name] = text_mode def _busy_profile_name_for_source(self, source: SessionSource) -> Optional[str]: """Return the routed profile whose busy policy applies, if any.""" @@ -381,46 +283,42 @@ class GatewayConfigLoadersMixin: name = "" return name or None - def _effective_busy_input_mode(self, source: SessionSource) -> str: - """Resolve busy input mode from the routed profile startup snapshot.""" - fallback = getattr(self, "_busy_input_mode", "interrupt") + def _effective_busy_mode(self, source: SessionSource, attr: str) -> str: + """Busy mode from the routed profile snapshot (``attr``: ``_busy_input_mode`` / ``_busy_text_mode``).""" + fallback = getattr(self, attr, "interrupt") profile_name = self._busy_profile_name_for_source(source) if not profile_name: return fallback - modes = getattr(self, "_busy_input_modes_by_profile", None) + modes = getattr(self, attr + "s_by_profile", None) return modes.get(profile_name, fallback) if isinstance(modes, dict) else fallback + def _effective_busy_input_mode(self, source: SessionSource) -> str: + """Resolve busy input mode from the routed profile startup snapshot.""" + return self._effective_busy_mode(source, "_busy_input_mode") + def _effective_busy_text_mode(self, source: SessionSource) -> str: """Resolve legacy busy text mode from the routed profile snapshot.""" - fallback = getattr(self, "_busy_text_mode", "interrupt") - profile_name = self._busy_profile_name_for_source(source) - if not profile_name: - return fallback - modes = getattr(self, "_busy_text_modes_by_profile", None) - return modes.get(profile_name, fallback) if isinstance(modes, dict) else fallback + return self._effective_busy_mode(source, "_busy_text_mode") @staticmethod - def _load_restart_drain_timeout() -> float: - """Load graceful gateway restart/stop drain timeout in seconds.""" - from gateway.run import _load_gateway_runtime_config - raw = os.getenv("HERMES_RESTART_DRAIN_TIMEOUT", "").strip() - if not raw: - cfg = _load_gateway_runtime_config() - raw = str(cfg_get(cfg, "agent", "restart_drain_timeout", default="") or "").strip() + def _warn_unparsable_timeout(cfg_key: str, raw: object, default: float) -> None: + """Warn when a supplied timeout value is not a number (the parser already fell back to ``default``).""" + try: + float(raw) + except (TypeError, ValueError): + logger.warning("Invalid %s '%s', using default %.0fs", cfg_key, raw, default) + + @classmethod + def _load_restart_drain_timeout(cls) -> float: + """Graceful gateway restart/stop drain timeout in seconds.""" + raw = cls._env_or_cfg_str("HERMES_RESTART_DRAIN_TIMEOUT", "agent", "restart_drain_timeout") value = parse_restart_drain_timeout(raw) if raw and value == DEFAULT_GATEWAY_RESTART_DRAIN_TIMEOUT: - try: - float(raw) - except (TypeError, ValueError): - logger.warning( - "Invalid restart_drain_timeout '%s', using default %.0fs", - raw, - DEFAULT_GATEWAY_RESTART_DRAIN_TIMEOUT, - ) + cls._warn_unparsable_timeout("restart_drain_timeout", raw, DEFAULT_GATEWAY_RESTART_DRAIN_TIMEOUT) return value - @staticmethod - def _load_env_or_agent_cfg_timeout(env_var: str, cfg_key: str, parse, default: float) -> float: + @classmethod + def _load_env_or_agent_cfg_timeout(cls, env_var: str, cfg_key: str, parse, default: float) -> float: """Env var (non-empty) else ``agent.``; warn once when a supplied value fails to parse. ``0`` is a valid value; the parser falls back to ``default`` on garbage.""" @@ -429,19 +327,15 @@ class GatewayConfigLoadersMixin: if env_raw is not None and str(env_raw).strip() != "": raw: object = env_raw else: - cfg = _load_gateway_runtime_config() - raw = cfg_get(cfg, "agent", cfg_key, default=None) + raw = cfg_get(_load_gateway_runtime_config(), "agent", cfg_key, default=None) value = parse(raw) if raw is not None and str(raw).strip() != "": - try: - float(raw) - except (TypeError, ValueError): - logger.warning("Invalid %s '%s', using default %.0fs", cfg_key, raw, default) + cls._warn_unparsable_timeout(cfg_key, raw, default) return value @classmethod def _load_restart_after_turn_timeout(cls) -> float: - """Load in-band restart wait-for-idle timeout in seconds.""" + """In-band restart wait-for-idle timeout in seconds.""" return cls._load_env_or_agent_cfg_timeout( "HERMES_RESTART_AFTER_TURN_TIMEOUT", "restart_after_turn_timeout", parse_restart_after_turn_timeout, DEFAULT_GATEWAY_RESTART_AFTER_TURN_TIMEOUT, @@ -449,105 +343,66 @@ class GatewayConfigLoadersMixin: @classmethod def _load_cron_drain_timeout(cls) -> float: - """Load the cron-only floor under the stop()/drain wait.""" + """The cron-only floor under the stop()/drain wait.""" return cls._load_env_or_agent_cfg_timeout( "HERMES_CRON_DRAIN_TIMEOUT", "cron_drain_timeout", parse_cron_drain_timeout, DEFAULT_GATEWAY_CRON_DRAIN_TIMEOUT, ) - @staticmethod - def _load_signal_interrupt_grace_timeout() -> float: - """Load the unexpected-signal post-interrupt grace in seconds.""" + @classmethod + def _load_signal_interrupt_grace_timeout(cls) -> float: + """``gateway.signal_interrupt_grace_timeout``: unexpected-signal post-interrupt grace in seconds.""" from gateway.run import _load_gateway_runtime_config - cfg = _load_gateway_runtime_config() - raw = cfg_get( - cfg, - "gateway", - "signal_interrupt_grace_timeout", - default=None, - ) + raw = cfg_get(_load_gateway_runtime_config(), "gateway", "signal_interrupt_grace_timeout", default=None) value = parse_signal_interrupt_grace_timeout(raw) if raw is not None and raw != "": - try: - float(raw) - except (TypeError, ValueError): - logger.warning( - "Invalid signal_interrupt_grace_timeout '%s', using default %.0fs", - raw, - DEFAULT_GATEWAY_SIGNAL_INTERRUPT_GRACE_TIMEOUT, - ) + cls._warn_unparsable_timeout( + "signal_interrupt_grace_timeout", raw, DEFAULT_GATEWAY_SIGNAL_INTERRUPT_GRACE_TIMEOUT + ) return value def _post_interrupt_grace_timeout(self) -> float: - """Return the grace before teardown after forcibly interrupting agents.""" - if ( - getattr(self, "_signal_initiated_shutdown", False) - and not getattr(self, "_restart_requested", False) - ): - return max( - 0.0, - float( - getattr( - self, - "_signal_interrupt_grace_timeout", - DEFAULT_GATEWAY_SIGNAL_INTERRUPT_GRACE_TIMEOUT, - ) - ), - ) + """Grace before teardown after forcibly interrupting agents (longer on an unexpected signal).""" + if getattr(self, "_signal_initiated_shutdown", False) and not getattr(self, "_restart_requested", False): + grace = getattr(self, "_signal_interrupt_grace_timeout", DEFAULT_GATEWAY_SIGNAL_INTERRUPT_GRACE_TIMEOUT) + return max(0.0, float(grace)) return DEFAULT_GATEWAY_POST_INTERRUPT_GRACE_TIMEOUT @staticmethod def _load_background_notifications_mode() -> str: - """Load background process notification mode from config or env var.""" + """Background process notification mode from env/config (default ``concise``).""" from gateway.run import _load_gateway_runtime_config mode = os.getenv("HERMES_BACKGROUND_NOTIFICATIONS", "") if not mode: - cfg = _load_gateway_runtime_config() - raw = cfg_get(cfg, "display", "background_process_notifications") + raw = cfg_get(_load_gateway_runtime_config(), "display", "background_process_notifications") if raw is False: mode = "off" elif raw not in {None, ""}: mode = str(raw) mode = (mode or "concise").strip().lower() - valid = {"concise", "all", "result", "error", "off"} - if mode not in valid: - logger.warning( - "Unknown background_process_notifications '%s', defaulting to 'concise'", - mode, - ) + if mode not in {"concise", "all", "result", "error", "off"}: + logger.warning("Unknown background_process_notifications '%s', defaulting to 'concise'", mode) return "concise" return mode @staticmethod def _load_provider_routing() -> dict: - """Load OpenRouter provider routing preferences from config.yaml.""" + """OpenRouter provider routing preferences (canonical fail-open loader: managed overlay + ${VAR}).""" from gateway.run import _load_gateway_runtime_config try: - # Canonical gateway loader (fail-open): managed overlay + ${VAR} - # expansion now apply to provider_routing too. - cfg = _load_gateway_runtime_config() - return cfg.get("provider_routing", {}) or {} + return _load_gateway_runtime_config().get("provider_routing", {}) or {} except Exception: - pass - return {} + return {} @staticmethod def _load_fallback_model() -> list | None: - """Load fallback provider chain from config.yaml. - - Merges ``fallback_providers`` (kept first) with legacy ``fallback_model`` entries. - """ + """Fallback chain: ``fallback_providers`` (kept first) merged with legacy ``fallback_model``.""" from gateway.run import _load_gateway_runtime_config try: - # Canonical gateway loader (fail-open): managed overlay + ${VAR} - # expansion now apply to the fallback chain too. - cfg = _load_gateway_runtime_config() - fb = get_fallback_chain(cfg) - if fb: - return fb + # Canonical gateway loader (fail-open): managed overlay + ${VAR} expansion apply here too. + return get_fallback_chain(_load_gateway_runtime_config()) or None except Exception: - pass - return None + return None def _refresh_fallback_model(self) -> list | None: """Re-read fallback_providers from disk for the next agent create/reuse. @@ -567,24 +422,16 @@ class GatewayConfigLoadersMixin: # loader would return {} on a torn mid-edit write and WIPE the last known-good chain. # The overlay/expansion below fixes the managed-scope/${VAR} drift without losing that. cfg = read_user_config_raw(cfg_path) - try: + with suppress(Exception): from hermes_cli import managed_scope cfg = managed_scope.apply_managed_overlay(cfg) - except Exception: - pass - try: + with suppress(Exception): from hermes_cli.config import _expand_env_vars expanded = _expand_env_vars(cfg) if isinstance(expanded, dict): cfg = expanded - except Exception: - pass except Exception: - # Transient failure — keep last known-good chain. - logger.debug( - "fallback_providers refresh: config.yaml read failed; " - "keeping last known-good chain", exc_info=True, - ) + logger.debug("fallback_providers refresh: config.yaml read failed; keeping last known-good chain", exc_info=True) return self._fallback_model self._fallback_model = get_fallback_chain(cfg) or None return self._fallback_model @@ -601,10 +448,7 @@ class GatewayConfigLoadersMixin: return new_chain = list(chain or []) rate_limited_until = getattr(agent, "_rate_limited_until", 0) or 0 - if ( - getattr(agent, "_fallback_activated", False) - and rate_limited_until > time.monotonic() - ): + if getattr(agent, "_fallback_activated", False) and rate_limited_until > time.monotonic(): return old_chain = list(getattr(agent, "_fallback_chain", []) or []) agent._fallback_chain = new_chain diff --git a/gateway/run_goals.py b/gateway/run_goals.py index f48ccbbe9d..eef4a70442 100644 --- a/gateway/run_goals.py +++ b/gateway/run_goals.py @@ -7,13 +7,13 @@ so ``patch("gateway.run.X")`` keeps intercepting them at call time. from __future__ import annotations -import logging -from typing import TYPE_CHECKING import asyncio +import logging import time from contextlib import suppress +from typing import TYPE_CHECKING, Any + from gateway.platforms.base import MessageEvent, MessageType -from typing import Any if TYPE_CHECKING: # string annotations only; never imported at runtime (cycle) from gateway.run import GatewayRunner, TurnRunner # noqa: F401 @@ -25,14 +25,12 @@ logger = logging.getLogger("gateway.run") class GatewayGoalsMixin: """Goal/heartbeat continuation, post-turn hooks and loop-wakeup watcher methods for GatewayRunner.""" - # ──────────────────────────────────────────────────────────────── - # /goal — persistent cross-turn goals (Ralph-style loop) - # ──────────────────────────────────────────────────────────────── + # ── /goal — persistent cross-turn goals (Ralph-style loop) ────────── def _goal_max_turns_from_config(self) -> int: - """Resolve the configured /goal turn budget for gateway sessions. + """Resolve the configured /goal turn budget. - GatewayRunner.config is a GatewayConfig dataclass, not the full user config mapping, so - top-level blocks such as ``goals`` are only reachable via hermes_cli.config.load_config(). + GatewayRunner.config is a GatewayConfig dataclass, not the full user config, so the + top-level ``goals`` block is only reachable via hermes_cli.config.load_config(). """ try: goals_cfg = ( @@ -51,10 +49,9 @@ class GatewayGoalsMixin: async def _warm_goals_session_db(self, label: str) -> None: """Warm the goals SessionDB cache off-loop (best-effort). - A cold cache runs the state.db init on the loop thread and freezes the loop for the init - duration. The executor hop keeps the profile home override alive under multiplex, so the - warm cache belongs to the caller's profile. On failure the caller falls back to the - bootstrap windows, so a dropped warm-up is a bounded stall, never a crash. + A cold cache runs the state.db init on the loop thread and freezes the loop. The executor + hop keeps the profile home override alive under multiplex. On failure the caller falls + back to the bootstrap windows, so a dropped warm-up is a bounded stall, never a crash. """ try: from hermes_cli.goals import _get_session_db as _warm_goals_db @@ -66,15 +63,14 @@ class GatewayGoalsMixin: async def _session_entry_for_manager(self, event: "MessageEvent", label: str): """Session entry for a /goal or /heartbeat manager, or None when lookup fails. - Warms the SessionDB cache off-loop first: a cold cache freezes the loop for the init - duration and drops the first write while the reply claims it was set. Internal events look - the session up WITHOUT touching activity so they never advance the idle/daily reset clock. + Warms the SessionDB cache first (a cold cache drops the first write while the reply + claims it was set). Internal events never touch activity, so they don't advance the + idle/daily reset clock. """ await self._warm_goals_session_db(label) try: session_entry = await self.async_session_store.get_or_create_session( - event.source, - touch_activity=not bool(getattr(event, "internal", False)), + event.source, touch_activity=not bool(getattr(event, "internal", False)), ) except Exception as exc: logger.debug("%s: session lookup failed: %s", label, exc) @@ -108,6 +104,11 @@ class GatewayGoalsMixin: return None, None return HeartbeatManager(session_id=session_entry.session_id), session_entry + @staticmethod + def _synthetic_prompt_event(source: Any, text: str, *, internal: bool = False) -> MessageEvent: + """Build the TEXT event used to inject a goal/heartbeat/loop prompt into a session.""" + return MessageEvent(text=text, message_type=MessageType.TEXT, source=source, internal=internal) + def _register_heartbeat_watch(self, quick_key: str, source: Any, session_id: str) -> None: """Track a session with an active heartbeat and start the poller. @@ -128,6 +129,32 @@ class GatewayGoalsMixin: if watch: watch.pop(quick_key, None) + async def _heartbeat_poll_once(self, watch: dict) -> None: + """One heartbeat poll pass: enqueue every due prompt of a non-busy watched session.""" + # Warm the cache off-loop once per poll: this only covers the degraded path where the + # /heartbeat command's own warm-up failed. + await self._warm_goals_session_db("heartbeat poll") + for quick_key, (source, session_id) in list(watch.items()): + try: + # Busy sessions coalesce their tick to the next idle poll. + if quick_key in self._running_agents: + continue + from hermes_cli.heartbeat import HeartbeatManager + + mgr = HeartbeatManager(session_id=session_id) + if not mgr.has_heartbeat(): + watch.pop(quick_key, None) + continue + prompt = mgr.due_prompt() + if not prompt: + continue + adapter = self._adapter_for_source(source) + if adapter is None: + continue + self._enqueue_fifo(quick_key, self._synthetic_prompt_event(source, prompt), adapter) + except Exception as exc: + logger.debug("heartbeat poll for %s failed: %s", quick_key, exc) + def _start_heartbeat_poller(self) -> None: """Start the single gateway-wide heartbeat poll task (idempotent).""" existing = getattr(self, "_heartbeat_poll_task", None) @@ -140,46 +167,14 @@ class GatewayGoalsMixin: while True: await asyncio.sleep(POLL_SECONDS) watch = getattr(self, "_heartbeat_watch", None) - if not watch: - continue - # Warm the cache off-loop once per poll. A watch can only be registered through the - # warmed /heartbeat command, so this covers only the degraded path where that warm- - # up failed. - await self._warm_goals_session_db("heartbeat poll") - for quick_key, (source, session_id) in list(watch.items()): - try: - # Busy sessions coalesce their tick to the next idle poll. - if quick_key in self._running_agents: - continue - from hermes_cli.heartbeat import HeartbeatManager - - mgr = HeartbeatManager(session_id=session_id) - if not mgr.has_heartbeat(): - watch.pop(quick_key, None) - continue - prompt = mgr.due_prompt() - if not prompt: - continue - adapter = self._adapter_for_source(source) - if adapter is None: - continue - hb_event = MessageEvent( - text=prompt, - message_type=MessageType.TEXT, - source=source, - message_id=None, - channel_prompt=None, - ) - self._enqueue_fifo(quick_key, hb_event, adapter) - except Exception as exc: - logger.debug("heartbeat poll for %s failed: %s", quick_key, exc) + if watch: + await self._heartbeat_poll_once(watch) try: task = asyncio.create_task(_poll_loop()) self._heartbeat_poll_task = task - # PERMANENT once started (an infinite while-True loop, no exit condition) — same as a - # _spawn_supervised watcher. Tag it so _scale_to_zero_has_live_background_work() doesn't - # treat a gateway with an active heartbeat watch as busy forever. + # PERMANENT once started (infinite loop) — same as a _spawn_supervised watcher. Tag it + # so _scale_to_zero_has_live_background_work() doesn't treat the gateway as busy forever. task._hermes_supervised_watcher = True # type: ignore[attr-defined] _bg = getattr(self, "_background_tasks", None) if _bg is not None: @@ -194,17 +189,14 @@ class GatewayGoalsMixin: if not adapter: logger.debug("goal continuation: no adapter for %s", getattr(source, "platform", None)) return - try: metadata = self._thread_metadata_for_source(source) except Exception: metadata = None - result = await adapter.send(source.chat_id, message, metadata=metadata) if result is not None and not getattr(result, "success", True): logger.warning( - "goal continuation: status send failed: %s", - getattr(result, "error", "unknown error"), + "goal continuation: status send failed: %s", getattr(result, "error", "unknown error"), ) async def _defer_goal_status_notice_after_delivery(self, source: Any, message: str) -> None: @@ -236,11 +228,7 @@ class GatewayGoalsMixin: active = getattr(adapter, "_active_sessions", {}).get(session_key) if active is not None: generation = getattr(active, "_hermes_run_generation", None) - adapter.register_post_delivery_callback( - session_key, - _deliver, - generation=generation, - ) + adapter.register_post_delivery_callback(session_key, _deliver, generation=generation) return except Exception as exc: logger.debug("goal continuation: post-delivery callback registration failed: %s", exc) @@ -248,14 +236,9 @@ class GatewayGoalsMixin: await _deliver() async def _post_turn_goal_continuation( - self, - *, - session_entry: Any, - source: Any, - final_response: str, + self, *, session_entry: Any, source: Any, final_response: str, ) -> None: - """Run the goal judge after a gateway turn and, if still active, enqueue a continuation - prompt for the same session. + """Run the goal judge after a gateway turn and, if still active, enqueue a continuation. Called at turn boundary AFTER delivery. Uses the adapter's pending-message/FIFO machinery so a simultaneous real user message is handled by the same queue and takes priority. @@ -271,9 +254,8 @@ class GatewayGoalsMixin: return max_turns = self._goal_max_turns_from_config() - - # Warm the SessionDB cache off-loop: a cold cache runs the state.db init on the loop thread - # at the turn boundary; a slow init can drop the goal read and silently end the goal loop. + # Cold cache at the turn boundary: a slow state.db init on the loop thread can drop the + # goal read and silently end the goal loop. await self._warm_goals_session_db("goal continuation") mgr = GoalManager(session_id=sid, default_max_turns=max_turns) @@ -286,23 +268,19 @@ class GatewayGoalsMixin: except Exception: _bg_procs = None - # evaluate_after_turn calls judge_goal(), a synchronous HTTP request to the auxiliary LLM; - # on the event-loop thread it blocks Discord heartbeats 10-40 s and flaps connections, so it - # is offloaded to a thread-pool executor. _run_in_executor_with_context (not bare - # run_in_executor): the profile secret scope and aux runtime context are contextvars; a - # default-executor hop drops them and aux credential resolution fails under multiplexing. + # evaluate_after_turn → judge_goal() is a synchronous aux-LLM HTTP call; on the loop thread + # it blocks Discord heartbeats 10-40 s. _run_in_executor_with_context (not bare + # run_in_executor) carries the profile secret scope / aux runtime contextvars, without which + # aux credential resolution fails under multiplexing. decision = await self._run_in_executor_with_context( lambda: mgr.evaluate_after_turn( - final_response or "", - user_initiated=True, - background_processes=_bg_procs, + final_response or "", user_initiated=True, background_processes=_bg_procs, ), ) msg = decision.get("message") or "" - # Defer the status line until after the adapter has delivered the agent's visible final - # response. The judge runs after the response is produced but before BasePlatformAdapter - # sends it, so sending here would show "✓ Goal achieved" before the answer itself. + # Status line is deferred until the adapter has delivered the visible final response, + # otherwise "✓ Goal achieved" would show before the answer itself. if msg and source is not None: await self._defer_goal_status_notice_after_delivery(source, msg) @@ -313,59 +291,41 @@ class GatewayGoalsMixin: if not prompt or source is None: return - # Enqueue via the adapter's FIFO so a user message already in - # flight preempts the continuation naturally. + # Enqueue via the adapter's FIFO so a user message already in flight preempts naturally. try: adapter = self._adapter_for_source(source) _quick_key = self._session_key_for_source(source) if adapter and _quick_key: - cont_event = MessageEvent( - text=prompt, - message_type=MessageType.TEXT, - source=source, - message_id=None, - channel_prompt=None, - ) - self._enqueue_fifo(_quick_key, cont_event, adapter) + self._enqueue_fifo(_quick_key, self._synthetic_prompt_event(source, prompt), adapter) except Exception as exc: logger.debug("goal continuation: enqueue failed: %s", exc) async def _run_post_turn_hooks( - self, - *, - agent_result: Any, - source: Any, - is_internal: bool, - event: Any = None, + self, *, agent_result: Any, source: Any, is_internal: bool, event: Any = None, ) -> None: """Run goal and loop bookkeeping after an agent turn returns.""" final_text = self._final_text_for_post_turn_hooks(agent_result, event) try: session_entry = await self.async_session_store.get_or_create_session( - source, - touch_activity=not is_internal, + source, touch_activity=not is_internal, ) except Exception as exc: logger.debug("post-turn session resolution failed: %s", exc) return - # Empty interrupted/errored responses must not drive /goal, but an - # in-flight /loop tick still needs to be released and rescheduled. + # Empty interrupted/errored responses must not drive /goal, but an in-flight /loop tick + # still needs to be released and rescheduled. if final_text.strip(): try: await self._post_turn_goal_continuation( - session_entry=session_entry, - source=source, - final_response=final_text, + session_entry=session_entry, source=source, final_response=final_text, ) except Exception as exc: logger.debug("goal continuation hook failed: %s", exc) try: await self._post_turn_loop_completion( - session_entry=session_entry, - source=source, - final_response=final_text, + session_entry=session_entry, source=source, final_response=final_text, ) except Exception as exc: logger.debug("loop completion hook failed: %s", exc) @@ -374,7 +334,7 @@ class GatewayGoalsMixin: def _final_text_for_post_turn_hooks(agent_result, event=None) -> str: """Text for /goal and /loop after a gateway turn. - Streamed turns return None from _handle_message_with_agent (already_sent). The delivered + Streamed turns return None from _handle_message_with_agent (already_sent); the delivered reply is stashed on the event so those hooks still see it. """ text = "" @@ -390,11 +350,7 @@ class GatewayGoalsMixin: return text async def _post_turn_loop_completion( - self, - *, - session_entry: Any, - source: Any, - final_response: str, + self, *, session_entry: Any, source: Any, final_response: str, ) -> None: """Complete a /loop wakeup tick after a gateway turn. @@ -412,8 +368,7 @@ class GatewayGoalsMixin: if not sid: return - # Warm the SessionDB cache off-loop: a cold cache at the turn boundary stalls the loop for - # the init duration and can drop the tick-completion write (the /goal continuation seam). + # Cold cache at the turn boundary can drop the tick-completion write (see /goal above). await self._warm_goals_session_db("loop completion") mgr = LoopManager(session_id=sid) @@ -429,6 +384,69 @@ class GatewayGoalsMixin: if msg and source is not None: await self._defer_goal_status_notice_after_delivery(source, msg) + async def _loop_wakeup_fire_one(self, sid: str, state: Any, now: float, warned_no_route: set) -> None: + """Inject one due /loop wakeup into its session, applying every deferral rule.""" + from hermes_cli.loops import LoopManager, goal_blocks_loop_tick + + if state.awaiting_response or now < state.next_due_at: + return + route = state.route or {} + platform_name = route.get("platform", "") + chat_id = route.get("chat_id", "") + if not platform_name or not chat_id: + return # CLI / TUI-owned loop — their own schedulers drive it. + adapter = next((a for p, a in self.adapters.items() if p.value == platform_name), None) + if adapter is None: + if sid not in warned_no_route: + warned_no_route.add(sid) + logger.debug( + "loop wakeup: no adapter for platform %r (session %s)", platform_name, sid, + ) + return + + # Build the source + session key to check business. + source = self._build_process_event_source({ + "session_key": "", + "platform": platform_name, + "chat_id": chat_id, + "chat_type": route.get("chat_type", ""), + "thread_id": route.get("thread_id", ""), + "user_id": route.get("user_id", ""), + "user_name": route.get("user_name", ""), + }) + if source is None: + return + try: + session_key = self._session_key_for_source(source) + except Exception: + session_key = None + if session_key and session_key in self._running_agents: + return # busy — stays due, next scan retries + if goal_blocks_loop_tick(sid): + return + + mgr = LoopManager(session_id=sid) + if not mgr.is_due(now): + return + wakeup = mgr.fire_tick() + if not wakeup: + return + try: + logger.info( + "loop wakeup #%s — injecting for %s chat=%s thread=%s", + mgr.state.ticks_fired if mgr.state else "?", + platform_name, source.chat_id, source.thread_id, + ) + await adapter.handle_message(self._synthetic_prompt_event(source, wakeup, internal=True)) + # Slash-command loops dispatch through the command path and never hit the post-turn + # completion hook — complete the tick immediately (caps + scheduling). + if wakeup.lstrip().startswith("/"): + mgr.complete_tick("") + except Exception as exc: + logger.warning("loop wakeup injection failed for %s: %s", sid, exc) + with suppress(Exception): + mgr.abandon_tick() + async def _loop_wakeup_watcher(self, interval: float = 15.0) -> None: """Fire due /loop wakeups for idle gateway sessions. @@ -443,11 +461,7 @@ class GatewayGoalsMixin: warned_no_route: set = set() while self._running: try: - from hermes_cli.loops import ( - LoopManager, - goal_blocks_loop_tick, - list_active_loops, - ) + from hermes_cli.loops import list_active_loops # Warm the cache off-loop once per scan: the scan reads every persisted loop, so a # cold cache would run the state.db init on the loop thread before the first read. @@ -455,78 +469,7 @@ class GatewayGoalsMixin: now = time.time() for sid, state in list_active_loops(): - if state.awaiting_response or now < state.next_due_at: - continue - route = state.route or {} - platform_name = route.get("platform", "") - chat_id = route.get("chat_id", "") - if not platform_name or not chat_id: - # CLI / TUI-owned loop — their own schedulers drive it. - continue - adapter = None - for p, a in self.adapters.items(): - if p.value == platform_name: - adapter = a - break - if adapter is None: - if sid not in warned_no_route: - warned_no_route.add(sid) - logger.debug( - "loop wakeup: no adapter for platform %r (session %s)", - platform_name, sid, - ) - continue - - # Build the source + session key to check business. - evt_stub = { - "session_key": "", - "platform": platform_name, - "chat_id": chat_id, - "chat_type": route.get("chat_type", ""), - "thread_id": route.get("thread_id", ""), - "user_id": route.get("user_id", ""), - "user_name": route.get("user_name", ""), - } - source = self._build_process_event_source(evt_stub) - if source is None: - continue - try: - session_key = self._session_key_for_source(source) - except Exception: - session_key = None - if session_key and session_key in self._running_agents: - continue # busy — stays due, next scan retries - if goal_blocks_loop_tick(sid): - continue - - mgr = LoopManager(session_id=sid) - if not mgr.is_due(now): - continue - wakeup = mgr.fire_tick() - if not wakeup: - continue - try: - synth_event = MessageEvent( - text=wakeup, - message_type=MessageType.TEXT, - source=source, - internal=True, - ) - logger.info( - "loop wakeup #%s — injecting for %s chat=%s thread=%s", - mgr.state.ticks_fired if mgr.state else "?", - platform_name, source.chat_id, source.thread_id, - ) - await adapter.handle_message(synth_event) - # Slash-command loops dispatch through the command - # path and never hit the post-turn completion hook — - # complete the tick immediately (caps + scheduling). - if wakeup.lstrip().startswith("/"): - mgr.complete_tick("") - except Exception as exc: - logger.warning("loop wakeup injection failed for %s: %s", sid, exc) - with suppress(Exception): - mgr.abandon_tick() + await self._loop_wakeup_fire_one(sid, state, now, warned_no_route) except Exception as exc: logger.debug("loop wakeup watcher error: %s", exc) await asyncio.sleep(interval) diff --git a/gateway/run_inbound.py b/gateway/run_inbound.py index 8b014b1d57..72ced13593 100644 --- a/gateway/run_inbound.py +++ b/gateway/run_inbound.py @@ -21,9 +21,7 @@ from gateway.config import Platform from gateway.platforms.base import EphemeralReply, MessageEvent, MessageType from gateway.run_common import _UNSET from gateway.session import ( - SessionSource, - is_shared_multi_user_session, - neutralize_untrusted_inline_text, + SessionSource, is_shared_multi_user_session, neutralize_untrusted_inline_text ) from gateway.turn_lease import TurnLeaseTimeoutError from typing import Any, Dict, List, Optional, Tuple @@ -38,32 +36,107 @@ logger = logging.getLogger("gateway.run") class GatewayInboundMixin: """Inbound message pipeline (_handle_message, text/media preparation, durable-turn markers, plugin injection) for GatewayRunner.""" + def _hm_pre_gateway_dispatch_hook( + self, event: "MessageEvent", source: SessionSource + ) -> Optional["MessageEvent"]: + """Run the ``pre_gateway_dispatch`` plugin hook; None = drop, else the (maybe rewritten) event. + + Plugins may return ``{"action": "skip", "reason": ...}`` → drop; ``{"action": "rewrite", + "text": ...}`` → replace ``event.text``; ``{"action": "allow"}`` / None → normal dispatch. + Runs BEFORE auth so plugins can handle unauthorized senders without the pairing flow. + """ + try: + from hermes_cli.lifecycle import invoke_hook as _invoke_hook + _hook_results = _invoke_hook( + "pre_gateway_dispatch", + event=event, + gateway=self, + # getattr: bare-runner tests build GatewayRunner via object.__new__ without + # __init__; the hook must not fail dispatch over a missing attribute. + session_store=getattr(self, "session_store", None), + ) + except Exception as _hook_exc: + logger.warning("pre_gateway_dispatch invocation failed: %s", _hook_exc) + _hook_results = [] + + for _result in _hook_results: + if not isinstance(_result, dict): + continue + _action = _result.get("action") + if _action == "skip": + logger.info( + "pre_gateway_dispatch skip: reason=%s platform=%s chat=%s", + _result.get("reason"), source.platform.value if source.platform else "unknown", + source.chat_id or "unknown", + ) + return None + if _action == "rewrite": + _new_text = _result.get("text") + if isinstance(_new_text, str): + event = dataclasses.replace(event, text=_new_text) + break + if _action == "allow": + break + return event + + async def _hm_offer_pairing_code(self, source: SessionSource) -> None: + """DM an unauthorized sender a pairing code (rate-limited; groups never reach here).""" + platform_name = source.platform.value if source.platform else "unknown" + pairing_store = self._pairing_store_for(source) + if pairing_store is None: + logger.error("Cannot offer pairing code on %s: no pairing store", platform_name) + return + # Rate-limit ALL pairing responses (code or rejection) so a burst of DMs doesn't spam. + if pairing_store._is_rate_limited(platform_name, source.user_id): + return + code = pairing_store.generate_code(platform_name, source.user_id, source.user_name or "") + adapter = self._adapter_for_source(source) + if code: + if adapter: + store_profile = getattr(pairing_store, "profile", None) + profile_arg = ( + f"-p {store_profile} " + if isinstance(store_profile, str) and store_profile and store_profile != "default" + else "" + ) + await adapter.send( + source.chat_id, + f"Hi~ I don't recognize you yet!\n\n" + f"Here's your pairing code: `{code}`\n\n" + f"Ask the bot owner to run:\n" + f"`hermes {profile_arg}pairing approve " + f"{platform_name} {code}`" + ) + else: + if adapter: + await adapter.send( + source.chat_id, + "Too many pairing requests right now~ " + "Please try again later!" + ) + # Record rate limit so subsequent messages are silently ignored + pairing_store._record_rate_limit(platform_name, source.user_id) + async def _hm_admit_event( self, event: "MessageEvent" ) -> Optional[Tuple["MessageEvent", SessionSource, bool]]: - """Ingress gates for ``_handle_message``: leak guard, profile route, ignored channels, - startup-restore queueing, ``pre_gateway_dispatch`` hook, authorization/pairing. - - Returns ``None`` when the message is dropped, else ``(event, source, is_internal)`` — - the hook may have rewritten ``event``. - """ + """Ingress gates for ``_handle_message``; None when dropped, else ``(event, source, is_internal)`` + (the ``pre_gateway_dispatch`` hook may have rewritten ``event``).""" from gateway.run import _is_slack_ignored_channel source = event.source # 🔴 Cross-session leak guard. This per-message task was created via create_task(), which - # copies the spawning context: if a concurrent message had already bound its session via - # set_session_vars(), we inherited ITS HERMES_SESSION_* ContextVars, and until _set_session_env - # binds ours any subprocess would read the foreign identity (the _UNSET-strip guard can't - # help — the vars are set-to-foreign). Reset to _UNSET so that window strips safe instead. + # copies the spawning context: a concurrent message may already have bound ITS + # HERMES_SESSION_* ContextVars, and until _set_session_env binds ours any subprocess would + # read the foreign identity. Reset to _UNSET so that window strips safe instead. try: from gateway.session_context import reset_session_vars reset_session_vars() except Exception: logger.debug("reset_session_vars failed at handler entry", exc_info=True) - # Most adapters resolve profile routes in build_source(), before they hand us the event. A - # few internal/voice paths construct SessionSource directly, so resolve those here as the - # shared fail-closed ingress gate before authorization, hooks, or session side effects. + # Most adapters resolve profile routes in build_source(); internal/voice paths construct + # SessionSource directly, so resolve those here as the shared fail-closed ingress gate. if ( getattr(getattr(self, "config", None), "multiplex_profiles", False) and not getattr(source, "profile", None) @@ -76,9 +149,8 @@ class GatewayInboundMixin: except ProfileRouteRejected: source.profile_route_rejected = True - # SessionSource owns a strict boolean marker. Require the literal value - # so duck-typed test/internal sources with dynamic attributes are not - # mistaken for an explicit matched-route rejection. + # SessionSource owns a strict boolean marker; require the literal value so duck-typed + # test/internal sources with dynamic attributes are not mistaken for a rejection. if getattr(source, "profile_route_rejected", False) is True: logger.warning( "Dropping inbound message because its explicit profile route " @@ -86,8 +158,7 @@ class GatewayInboundMixin: ) return None - # Internal events (e.g. background-process completion notifications) - # are system-generated and must skip user authorization. + # Internal events (e.g. background-process notifications) skip user authorization. is_internal = bool(getattr(event, "internal", False)) # Ignored-channel guard runs FIRST — before startup-restore queueing, plugin hooks, auth, @@ -114,58 +185,20 @@ class GatewayInboundMixin: self._queue_startup_restore_event(event) return None - # scale-to-zero: stamp the gateway-scoped last-inbound clock (read by is_idle) for real - # user-originated inbound only. Internal/system events are NOT traffic — counting them - # would keep a genuinely idle gateway awake. if not is_internal: + # scale-to-zero: only real user-originated inbound stamps the last-inbound clock; + # counting internal/system events would keep a genuinely idle gateway awake. self._scale_to_zero_note_real_inbound() - - # pre_gateway_dispatch plugin hook (user-originated only). Plugins may return - # {"action": "skip", "reason": ...} -> drop; {"action": "rewrite", "text": ...} -> replace - # event.text; {"action": "allow"} / None -> normal dispatch. - # Runs BEFORE auth so plugins can handle unauthorized senders without the pairing flow. - if not is_internal: - try: - from hermes_cli.lifecycle import invoke_hook as _invoke_hook - _hook_results = _invoke_hook( - "pre_gateway_dispatch", - event=event, - gateway=self, - # getattr: bare-runner tests build GatewayRunner via object.__new__ without - # __init__; the hook must not fail dispatch over a missing attribute. - session_store=getattr(self, "session_store", None), - ) - except Exception as _hook_exc: - logger.warning("pre_gateway_dispatch invocation failed: %s", _hook_exc) - _hook_results = [] - - for _result in _hook_results: - if not isinstance(_result, dict): - continue - _action = _result.get("action") - if _action == "skip": - logger.info( - "pre_gateway_dispatch skip: reason=%s platform=%s chat=%s", - _result.get("reason"), - source.platform.value if source.platform else "unknown", - source.chat_id or "unknown", - ) - return None - if _action == "rewrite": - _new_text = _result.get("text") - if isinstance(_new_text, str): - event = dataclasses.replace(event, text=_new_text) - source = event.source - break - if _action == "allow": - break + event = self._hm_pre_gateway_dispatch_hook(event, source) + if event is None: + return None + source = event.source if is_internal: pass elif source.user_id is None: - # Messages with no user identity (Telegram service messages, channel forwards, anonymous - # admin posts, sender_chat) can't be paired but may be authorized via a chat-scoped - # allowlist (e.g. TELEGRAM_GROUP_ALLOWED_CHATS), so defer to _is_user_authorized. + # No user identity (Telegram service messages, channel forwards, anonymous admin posts, + # sender_chat): can't be paired but may be authorized via a chat-scoped allowlist. if not self._is_user_authorized_for_source(source): logger.debug("Ignoring message with no user_id from %s", source.platform.value) return None @@ -174,286 +207,230 @@ class GatewayInboundMixin: # In DMs: offer pairing code. In groups: silently ignore. if ( source.chat_type == "dm" - and self._get_unauthorized_dm_behavior( - source.platform, - profile=source.profile, - ) - == "pair" + and self._get_unauthorized_dm_behavior(source.platform, profile=source.profile) == "pair" ): - platform_name = source.platform.value if source.platform else "unknown" - pairing_store = self._pairing_store_for(source) - if pairing_store is None: - logger.error( - "Cannot offer pairing code on %s: no pairing store", - platform_name, - ) - return None - # Rate-limit ALL pairing responses (code or rejection) so a burst of DMs doesn't - # spam the user with repeated messages. - if pairing_store._is_rate_limited(platform_name, source.user_id): - return None - code = pairing_store.generate_code( - platform_name, source.user_id, source.user_name or "" - ) - if code: - adapter = self._adapter_for_source(source) - if adapter: - store_profile = getattr(pairing_store, "profile", None) - profile_arg = ( - f"-p {store_profile} " - if isinstance(store_profile, str) - and store_profile - and store_profile != "default" - else "" - ) - await adapter.send( - source.chat_id, - f"Hi~ I don't recognize you yet!\n\n" - f"Here's your pairing code: `{code}`\n\n" - f"Ask the bot owner to run:\n" - f"`hermes {profile_arg}pairing approve " - f"{platform_name} {code}`" - ) - else: - adapter = self._adapter_for_source(source) - if adapter: - await adapter.send( - source.chat_id, - "Too many pairing requests right now~ " - "Please try again later!" - ) - # Record rate limit so subsequent messages are silently ignored - pairing_store._record_rate_limit(platform_name, source.user_id) + await self._hm_offer_pairing_code(source) return None return event, source, is_internal + def _hm_estop_turn_allowed(self, event: "MessageEvent", source: SessionSource) -> bool: + """Whether a turn may bypass the global emergency stop. + + Pause blocks NEW agent turns, never running work or control traffic: recognized slash + commands (/status, /approve, ... and /pause off as the in-band resume path) and replies + owned by in-flight work — pending update prompt, steering a running session, pending + slash-confirm, dangerous-command approval — all pass through. + """ + try: + _estop_cmd = event.get_command() + except Exception: + _estop_cmd = None + if _estop_cmd: + try: + from hermes_cli.commands import resolve_command as _resolve_estop_cmd + if _resolve_estop_cmd(_estop_cmd) is not None: + return True + except Exception: + pass + try: + _estop_key = self._session_key_for_source(source) + _estop_state = self._peek_session_state(_estop_key) + if _estop_state is not None and _estop_state.persistent.update_prompt_pending: + return True + if self._is_session_running(_estop_key): + # Steering / interrupting in-flight work (also covers pending clarify + tool + # approvals held by the running agent). + return True + from tools import slash_confirm as _estop_confirm_mod + if _estop_confirm_mod.get_pending(_estop_key): + return True + from tools.approval import has_blocking_approval as _estop_has_approval + if _estop_has_approval(_estop_key): + return True + except Exception: + pass + return False + def _hm_estop_gate( self, event: "MessageEvent", source: SessionSource, is_internal: bool ) -> Optional[str]: - """Return the global emergency-stop notice when this turn must be blocked, else None.""" - # Global emergency stop (`hermes pause`): new turns get a brief paused notice instead of an - # agent run. Placed after auth so unauthorized senders can't probe pause state. Pause blocks - # NEW agent turns, never running work or control traffic, so these pass through: internal - # events from IN-FLIGHT work; recognized slash commands (/status, /approve, ... and /pause off - # as the in-band resume path); replies owned by in-flight work — pending update prompt, - # clarify, slash-confirm, dangerous-command approval, or steering an already-running session. - if not is_internal: - try: - from agent.estop import paused_reply as _estop_paused_reply - _paused_notice = _estop_paused_reply() - except ImportError: - _paused_notice = None - if _paused_notice is not None: - _estop_allow = False - _estop_cmd = None - try: - _estop_cmd = event.get_command() - except Exception: - _estop_cmd = None - if _estop_cmd: - try: - from hermes_cli.commands import ( - resolve_command as _resolve_estop_cmd, - ) - _estop_allow = _resolve_estop_cmd(_estop_cmd) is not None - except Exception: - _estop_allow = False - if not _estop_allow: - try: - _estop_key = self._session_key_for_source(source) - _estop_state = self._peek_session_state(_estop_key) - if ( - _estop_state is not None - and _estop_state.persistent.update_prompt_pending - ): - _estop_allow = True - if not _estop_allow and self._is_session_running(_estop_key): - # Steering / interrupting in-flight work (also covers pending clarify + - # tool approvals held by the running agent). - _estop_allow = True - if not _estop_allow: - from tools import slash_confirm as _estop_confirm_mod - if _estop_confirm_mod.get_pending(_estop_key): - _estop_allow = True - if not _estop_allow: - from tools.approval import ( - has_blocking_approval as _estop_has_approval, - ) - if _estop_has_approval(_estop_key): - _estop_allow = True - except Exception: - pass - if not _estop_allow: - logger.info( - "Gateway turn paused by global emergency stop (platform=%s chat=%s)", - getattr(getattr(source, "platform", None), "value", "unknown"), - getattr(source, "chat_id", None) or "unknown", - ) - return _paused_notice + """Global emergency-stop (`hermes pause`) notice when this turn must be blocked, else None. + Placed after auth so unauthorized senders can't probe pause state.""" + if is_internal: + return None + try: + from agent.estop import paused_reply as _estop_paused_reply + _paused_notice = _estop_paused_reply() + except ImportError: + return None + if _paused_notice is None or self._hm_estop_turn_allowed(event, source): + return None + logger.info( + "Gateway turn paused by global emergency stop (platform=%s chat=%s)", + getattr(getattr(source, "platform", None), "value", "unknown"), + getattr(source, "chat_id", None) or "unknown", + ) + return _paused_notice + + @staticmethod + def _hm_write_update_response(response_text: str) -> Optional[str]: + """Atomically hand *response_text* to the detached update process; returns the OSError str.""" + from gateway.run import _hermes_home + response_path = _hermes_home / ".update_response" + prompt_path = _hermes_home / ".update_prompt.json" + try: + tmp = response_path.with_suffix(".tmp") + tmp.write_text(response_text, encoding="utf-8") + tmp.replace(response_path) + prompt_path.unlink(missing_ok=True) + except OSError as e: + return str(e) return None def _hm_update_prompt_reply( self, event: "MessageEvent", _quick_key: str, allow_gateway_control: bool ) -> Optional[str]: - """Consume a reply to a pending ``/update`` prompt; None when nothing was consumed.""" - from gateway.run import _hermes_home - # Route replies to a pending /update prompt back to the detached update process via - # .update_response. Recognized slash commands must bypass this or /new, /help etc. get - # silently consumed as update answers. + """Consume a reply to a pending ``/update`` prompt; None when nothing was consumed. + + Routes the reply to the detached update process via ``.update_response``. Recognized slash + commands must bypass this or /new, /help etc. get silently consumed as update answers. + """ _up_state = self._peek_session_state(_quick_key) - if ( + if not ( allow_gateway_control and _up_state is not None and _up_state.persistent.update_prompt_pending ): - raw = (event.text or "").strip() - # Accept /approve and /deny as shorthand for yes/no - cmd = event.get_command() - if cmd in {"approve", "yes"}: - response_text = "y" - elif cmd in {"deny", "no"}: - response_text = "n" + return None + raw = (event.text or "").strip() + # Accept /approve and /deny as shorthand for yes/no + cmd = event.get_command() + _recognized_cmd = None + if cmd in {"approve", "yes"}: + response_text = "y" + elif cmd in {"deny", "no"}: + response_text = "n" + else: + if cmd: + try: + from hermes_cli.commands import resolve_command as _resolve_update_cmd + _cmd_def = _resolve_update_cmd(cmd) + _recognized_cmd = _cmd_def.name if _cmd_def else None + except Exception: + _recognized_cmd = None + response_text = "" if _recognized_cmd else raw + if response_text: + err = self._hm_write_update_response(response_text) + if err is not None: + logger.warning("Failed to write update response: %s", err) + return f"✗ Failed to send response to update process: {err}" + _up_state.persistent.update_prompt_pending = False + label = response_text if len(response_text) <= 20 else response_text[:20] + "…" + return f"✓ Sent `{label}` to the update process." + # Recognized slash command during a pending update prompt: write a blank response so the + # detached update's ``_gateway_prompt`` returns the prompt's default (typically a safe + # "n" / skip) and exits instead of blocking on stdin until the watcher timeout. + if _recognized_cmd: + err = self._hm_write_update_response("") + if err is None: + logger.info( + "Recognized /%s during pending update prompt for %s; " + "cancelled prompt with default and dispatching command", + _recognized_cmd, + _quick_key, + ) else: - _recognized_cmd = None - if cmd: - try: - from hermes_cli.commands import resolve_command as _resolve_update_cmd - except Exception: - _resolve_update_cmd = None - if _resolve_update_cmd is not None: - try: - _cmd_def = _resolve_update_cmd(cmd) - _recognized_cmd = _cmd_def.name if _cmd_def else None - except Exception: - _recognized_cmd = None - response_text = "" if _recognized_cmd else raw - if response_text: - response_path = _hermes_home / ".update_response" - prompt_path = _hermes_home / ".update_prompt.json" - try: - tmp = response_path.with_suffix(".tmp") - tmp.write_text(response_text, encoding="utf-8") - tmp.replace(response_path) - prompt_path.unlink(missing_ok=True) - except OSError as e: - logger.warning("Failed to write update response: %s", e) - return f"✗ Failed to send response to update process: {e}" - _up_state.persistent.update_prompt_pending = False - label = response_text if len(response_text) <= 20 else response_text[:20] + "…" - return f"✓ Sent `{label}` to the update process." - # Recognized slash command during a pending update prompt: write a blank response so the - # detached update's ``_gateway_prompt`` returns the prompt's default (typically a safe - # "n" / skip) and exits instead of blocking on stdin until the watcher timeout. - if _recognized_cmd: - response_path = _hermes_home / ".update_response" - prompt_path = _hermes_home / ".update_prompt.json" - try: - tmp = response_path.with_suffix(".tmp") - tmp.write_text("", encoding="utf-8") - tmp.replace(response_path) - prompt_path.unlink(missing_ok=True) - logger.info( - "Recognized /%s during pending update prompt for %s; " - "cancelled prompt with default and dispatching command", - _recognized_cmd, - _quick_key, - ) - except OSError as e: - logger.warning( - "Failed to write cancel response for pending update prompt: %s", - e, - ) - _up_state.persistent.update_prompt_pending = False + logger.warning("Failed to write cancel response for pending update prompt: %s", err) + _up_state.persistent.update_prompt_pending = False return None async def _hm_clarify_reply( - self, - event: "MessageEvent", - source: SessionSource, - _quick_key: str, + self, event: "MessageEvent", source: SessionSource, _quick_key: str, allow_gateway_control: bool, ) -> Optional[str]: - """Intercept a reply to a pending clarify prompt; None when the message falls through.""" - # Intercept replies to a pending clarify: open-ended prompts and "Other" responses are free - # text; direct replies to multi-choice prompts are accepted too ("2" → second option). + """Intercept a reply to a pending clarify prompt; None when the message falls through. + + Open-ended prompts and "Other" responses are free text; direct replies to multi-choice + prompts are accepted too ("2" → second option). Resolved/retained replies return "" so + adapters that emit the agent's response don't double-post — the agent itself produces the + next user-facing message. + """ _clarify_mod = None try: from tools import clarify_gateway as _clarify_mod - _pending_clarify = _clarify_mod.get_pending_for_session( - _quick_key, include_choice_prompts=True, - ) + _pending_clarify = _clarify_mod.get_pending_for_session(_quick_key, include_choice_prompts=True) except Exception: _pending_clarify = None - if ( - allow_gateway_control - and _pending_clarify is not None - and _clarify_mod is not None - ): - _clarify_has_audio = bool(self._pending_event_audio_paths(event)) - _raw_clarify_reply = await self._prepare_clarify_reply_text(event) - if _clarify_has_audio and not _raw_clarify_reply: - logger.info( - "Gateway retained pending clarify after voice transcription " - "produced no usable text (session=%s, id=%s)", - _quick_key, - _pending_clarify.clarify_id, - ) - return "" - # Skip slash commands — the user wanted a command, not to answer the clarify. Leave it - # pending so they can retry; on timeout the agent unblocks with an empty response. - if _raw_clarify_reply and not _raw_clarify_reply.startswith("/"): - _text_outcome = _clarify_mod.attempt_text_response_for_session( - _quick_key, _raw_clarify_reply, - ) - if _text_outcome == _clarify_mod.TEXT_RESOLVED: - logger.info( - "Gateway intercepted clarify text response (session=%s, id=%s)", - _quick_key, _pending_clarify.clarify_id, - ) - # The clarify callback pauses the platform typing/status indicator while waiting - # so Slack users can type their answer. The active agent resumes as soon as this - # reply resolves the wait, so re-enable its indicator here too. - _clarify_adapter = self._adapter_for_source(source) - if _clarify_adapter: - try: - _clarify_adapter.resume_typing_for_chat(source.chat_id) - except Exception: - logger.debug( - "Failed to resume typing after clarify response", - exc_info=True, - ) - # Acknowledge with empty string so adapters that emit the agent's response don't - # double-post; the agent itself produces the next user-facing message. - return "" - if _text_outcome == _clarify_mod.TEXT_REJECTED_SELECTION: - # Selection-shaped but invalid (out-of-range number, bad comma-list): keep the - # clarify armed for retry — don't cancel, don't treat as an unrelated follow-up. - logger.info( - "Gateway retained pending clarify after invalid " - "selection attempt (session=%s, id=%s)", - _quick_key, _pending_clarify.clarify_id, - ) - return "" - if _text_outcome == _clarify_mod.TEXT_REJECTED_PROSE: - # Native-choice prompts deliberately reject unmatched prose so it can continue - # through normal busy-message routing. Release this clarify first: redirect() - # degrades to steer() while tools execute, and that steer cannot drain until - # the clarify tool returns. - _clarify_mod.resolve_gateway_clarify( - _pending_clarify.clarify_id, - "", - ) + if not (allow_gateway_control and _pending_clarify is not None and _clarify_mod is not None): + return None + _clarify_has_audio = bool(self._pending_event_audio_paths(event)) + _raw_clarify_reply = await self._prepare_clarify_reply_text(event) + if _clarify_has_audio and not _raw_clarify_reply: + logger.info( + "Gateway retained pending clarify after voice transcription " + "produced no usable text (session=%s, id=%s)", + _quick_key, + _pending_clarify.clarify_id, + ) + return "" + # Slash commands: the user wanted a command, not to answer the clarify. Leave it pending so + # they can retry; on timeout the agent unblocks with an empty response. + if not _raw_clarify_reply or _raw_clarify_reply.startswith("/"): + return None + _text_outcome = _clarify_mod.attempt_text_response_for_session(_quick_key, _raw_clarify_reply) + if _text_outcome == _clarify_mod.TEXT_RESOLVED: + logger.info( + "Gateway intercepted clarify text response (session=%s, id=%s)", + _quick_key, _pending_clarify.clarify_id, + ) + # The clarify callback pauses the platform typing/status indicator while waiting so + # Slack users can type; the active agent resumes now, so re-enable its indicator. + _clarify_adapter = self._adapter_for_source(source) + if _clarify_adapter: + try: + _clarify_adapter.resume_typing_for_chat(source.chat_id) + except Exception: + logger.debug("Failed to resume typing after clarify response", exc_info=True) + return "" + if _text_outcome == _clarify_mod.TEXT_REJECTED_SELECTION: + # Selection-shaped but invalid (out-of-range number, bad comma-list): keep the clarify + # armed for retry — don't cancel, don't treat as an unrelated follow-up. + logger.info( + "Gateway retained pending clarify after invalid " + "selection attempt (session=%s, id=%s)", + _quick_key, _pending_clarify.clarify_id, + ) + return "" + if _text_outcome == _clarify_mod.TEXT_REJECTED_PROSE: + # Native-choice prompts reject unmatched prose so it continues through normal busy + # routing. Release this clarify first: redirect() degrades to steer() while tools + # execute, and that steer cannot drain until the clarify tool returns. + _clarify_mod.resolve_gateway_clarify(_pending_clarify.clarify_id, "") return None + # Reply → choice for a pending slash-confirm prompt; the command spelling wins over the + # bang/slash-stripped free-text spelling. + _SLASH_CONFIRM_CMD_CHOICES = { + "approve": "once", "yes": "once", "ok": "once", "confirm": "once", + "always": "always", "remember": "always", + "cancel": "cancel", "no": "cancel", "deny": "cancel", "nevermind": "cancel", + } + _SLASH_CONFIRM_TEXT_CHOICES = { + "approve": "once", "approve once": "once", "once": "once", + "always": "always", "always approve": "always", + "cancel": "cancel", "nevermind": "cancel", "no": "cancel", + } + async def _hm_slash_confirm_reply( self, event: "MessageEvent", _quick_key: str, allow_gateway_control: bool ) -> Optional[str]: - """Resolve a reply to a pending slash-confirm prompt; None when the message falls through.""" - # Replies to a pending slash-confirm prompt (/reload-mcp etc.): /approve, /always, /cancel and - # short aliases. Anything else falls through — a stale pending confirm does NOT block other - # commands. A pending dangerous-command approval takes precedence: /approve there unblocks - # the waiting tool thread; slash-confirm only catches it when no tool approval is live. + """Resolve a reply to a pending slash-confirm prompt; None when the message falls through. + + Accepts /approve, /always, /cancel and short aliases. Anything else falls through — a stale + pending confirm does NOT block other commands. A pending dangerous-command approval takes + precedence: /approve there unblocks the waiting tool thread. + """ from tools import slash_confirm as _slash_confirm_mod _pending_confirm = _slash_confirm_mod.get_pending(_quick_key) _tool_approval_live = False @@ -462,220 +439,259 @@ class GatewayInboundMixin: _tool_approval_live = has_blocking_approval(_quick_key) except Exception: _tool_approval_live = False - if allow_gateway_control and _pending_confirm and not _tool_approval_live: - _raw_reply = (event.text or "").strip() - # Accept bang-prefixed replies (`!always`, `!cancel`) verbatim: Slack/Matrix show the - # `!` prefix (typed `/` is blocked in Slack threads) and adapters only rewrite - # `!` — confirm keywords aren't commands, so the `!` survives to here. - _norm_reply = _raw_reply.lstrip("!/").lower() - _cmd_reply = event.get_command() - _confirm_choice = None - if _cmd_reply in {"approve", "yes", "ok", "confirm"}: - _confirm_choice = "once" - elif _cmd_reply in {"always", "remember"}: - _confirm_choice = "always" - elif _cmd_reply in {"cancel", "no", "deny", "nevermind"}: - _confirm_choice = "cancel" - elif _norm_reply in {"approve", "approve once", "once"}: - _confirm_choice = "once" - elif _norm_reply in {"always", "always approve"}: - _confirm_choice = "always" - elif _norm_reply in {"cancel", "nevermind", "no"}: - _confirm_choice = "cancel" - if _confirm_choice is not None: - _resolved = await _slash_confirm_mod.resolve( - _quick_key, _pending_confirm.get("confirm_id"), _confirm_choice, - ) - return _resolved or "" - # Stale pending + unrelated command: the user moved on, so drop the pending state rather - # than let the confirm block normal usage indefinitely. - _slash_confirm_mod.clear_if_stale(_quick_key) + if not (allow_gateway_control and _pending_confirm and not _tool_approval_live): + return None + # Accept bang-prefixed replies (`!always`, `!cancel`) verbatim: Slack/Matrix show the `!` + # prefix (typed `/` is blocked in Slack threads) and adapters only rewrite + # `!` — confirm keywords aren't commands, so the `!` survives to here. + _norm_reply = (event.text or "").strip().lstrip("!/").lower() + _confirm_choice = ( + self._SLASH_CONFIRM_CMD_CHOICES.get(event.get_command()) + or self._SLASH_CONFIRM_TEXT_CHOICES.get(_norm_reply) + ) + if _confirm_choice is not None: + _resolved = await _slash_confirm_mod.resolve( + _quick_key, _pending_confirm.get("confirm_id"), _confirm_choice, + ) + return _resolved or "" + # Stale pending + unrelated command: the user moved on, so drop the pending state rather + # than let the confirm block normal usage indefinitely. + _slash_confirm_mod.clear_if_stale(_quick_key) return None - def _hm_evict_stale_running_agent(self, _quick_key: str) -> None: - """Evict a leaked/reaped ``_running_agents`` slot before the busy-session fast-path.""" + def _hm_evict_idle_stale_agent(self, _quick_key: str) -> None: + """Evict a leaked lock from a hung/crashed handler. + + With inactivity-based timeout active tasks can run for hours, so evict only when the agent + has been *idle* past the threshold (or has no activity tracker and its wall-clock age is + extreme). The pending sentinel is never evicted: it has no get_activity_summary(), so the + idle check would read inf and race the async setup path. + """ from gateway.run import _AGENT_PENDING_SENTINEL, _float_env - # Staleness eviction: detect leaked locks from hung/crashed handlers. With inactivity-based - # timeout active tasks can run for hours, so evict only when the agent has been *idle* past - # the threshold (or has no activity tracker and its wall-clock age is extreme). _raw_stale_timeout = _float_env("HERMES_AGENT_TIMEOUT", 1800) _quick_state = self._peek_session_state(_quick_key) _stale_ts = _quick_state.turn.started_ts if _quick_state else 0 - if _quick_state is not None and _quick_state.turn.agent is not None and _stale_ts: - _stale_age = time.time() - _stale_ts - _stale_agent = _quick_state.turn.agent - # Never evict the pending sentinel — it was just placed during async setup before the - # real agent exists. Sentinels have no get_activity_summary(), so the idle check would - # read inf >= timeout and evict them immediately, racing the setup path. - _stale_idle = float("inf") # assume idle if we can't check - _stale_detail = "" - _activity_summary_valid = False - if _stale_agent and hasattr(_stale_agent, "get_activity_summary"): - try: - _sa = _stale_agent.get_activity_summary() - from gateway.session_stall import ( - resolve_session_idle_seconds_from_activity, - ) - - _resolved_idle = resolve_session_idle_seconds_from_activity( - _sa if isinstance(_sa, dict) else None, - now=time.time(), - ) - if _resolved_idle is not None: - _stale_idle = _resolved_idle - _activity_summary_valid = True - _stale_detail = ( - f" | last_activity={_sa.get('last_activity_desc', 'unknown') if isinstance(_sa, dict) else 'unknown'} " - f"({_stale_idle:.0f}s ago) " - f"| iteration={_sa.get('api_call_count', 0) if isinstance(_sa, dict) else 0}/{_sa.get('max_iterations', 0) if isinstance(_sa, dict) else 0}" - ) - except Exception: - pass - # A valid activity clock is authoritative: total age alone never - # makes an actively progressing turn stale. The emergency wall TTL - # is only a fallback when the agent cannot report usable activity. - _wall_ttl = max(_raw_stale_timeout * 10, 7200) if _raw_stale_timeout > 0 else float("inf") - _should_evict = ( - _stale_agent is not _AGENT_PENDING_SENTINEL - and ( - ( - _activity_summary_valid - and _raw_stale_timeout > 0 - and _stale_idle >= _raw_stale_timeout - ) - or ( - not _activity_summary_valid - and _stale_age > _wall_ttl - ) - ) - ) - if _should_evict: - logger.warning( - "Evicting stale _running_agents entry for %s " - "(age: %.0fs, idle: %.0fs, timeout: %.0fs)%s", - _quick_key, _stale_age, _stale_idle, - _raw_stale_timeout, _stale_detail, - ) - self._invalidate_session_run_generation( - _quick_key, - reason="stale_running_agent_eviction", - ) - self._release_running_agent_state(_quick_key) - - # Durable-reaped guard. A session whose routing row was ended in state.db (``ws_orphan_reap`` - # / ``agent_close``) while the gateway lived keeps its in-memory turn slot, so the fast-path - # would queue every next message into the dead runtime. Evict the stale slot so the cold - # path re-attaches via ``get_or_create_session`` → ``reopen`` or creates a fresh session. - if self._is_session_running(_quick_key): + if _quick_state is None or _quick_state.turn.agent is None or not _stale_ts: + return + _stale_age = time.time() - _stale_ts + _stale_agent = _quick_state.turn.agent + _stale_idle = float("inf") # assume idle if we can't check + _stale_detail = "" + _activity_summary_valid = False + if _stale_agent and hasattr(_stale_agent, "get_activity_summary"): try: - _reap_store = getattr(self, "session_store", None) - # Use the public, lock-held accessors: peek_session_id resolves key -> session_id - # under the store lock, and returns a non-str on stubbed stores in bare test runners - # — both the isinstance() gate and the ``is True`` gate below keep this guard inert - # unless a real SessionStore answers. - _reap_peek = getattr(_reap_store, "peek_session_id", None) - _is_ended = getattr(_reap_store, "_is_session_ended_in_db", None) - _reap_sid = _reap_peek(_quick_key) if callable(_reap_peek) else None - if ( - isinstance(_reap_sid, str) - and _reap_sid - and callable(_is_ended) - and _is_ended(_reap_sid) is True - ): - logger.warning( - "Evicting stale _running_agents entry for %s — " - "durable session %s is ended (reaped) in state.db; " - "healing routing on next message (#99106)", - _quick_key, - _reap_sid, - ) - self._invalidate_session_run_generation( - _quick_key, - reason="reaped_session_eviction", - ) - self._release_running_agent_state(_quick_key) - except Exception: - logger.debug("reaped-session staleness check failed", exc_info=True) + _sa = _stale_agent.get_activity_summary() + from gateway.session_stall import resolve_session_idle_seconds_from_activity - async def _hm_handle_running_session_message( - self, event: "MessageEvent", source: SessionSource, _quick_key: str - ) -> Optional[str]: - """Fast-path for a message that arrives while this session's agent is running.""" - from gateway.run import ( - _AGENT_PENDING_SENTINEL, - _build_media_placeholder, - merge_pending_message_event, + _resolved_idle = resolve_session_idle_seconds_from_activity( + _sa if isinstance(_sa, dict) else None, now=time.time(), + ) + if _resolved_idle is not None: + _stale_idle = _resolved_idle + _activity_summary_valid = True + _sa_d = _sa if isinstance(_sa, dict) else {} + _stale_detail = ( + f" | last_activity={_sa_d.get('last_activity_desc', 'unknown')} " + f"({_stale_idle:.0f}s ago) " + f"| iteration={_sa_d.get('api_call_count', 0)}/{_sa_d.get('max_iterations', 0)}" + ) + except Exception: + pass + # A valid activity clock is authoritative: total age alone never makes an actively + # progressing turn stale. The emergency wall TTL is only a fallback when the agent cannot + # report usable activity. + _wall_ttl = max(_raw_stale_timeout * 10, 7200) if _raw_stale_timeout > 0 else float("inf") + _should_evict = _stale_agent is not _AGENT_PENDING_SENTINEL and ( + (_activity_summary_valid and _raw_stale_timeout > 0 and _stale_idle >= _raw_stale_timeout) + or (not _activity_summary_valid and _stale_age > _wall_ttl) ) - # PRIORITY handling when an agent is already running for this session. Default behavior is - # to interrupt immediately so user text/stop messages are handled with minimal latency. - # Exception: Telegram photo bursts arrive as near-simultaneous updates — do NOT interrupt - # for photo-only follow-ups; adapter-level batching absorbs them. - # Resolve the command once; each command's mid-run behavior is declared on its - # CommandDef (busy_policy / busy_handler in hermes_cli/commands.py) and dispatched via - # _dispatch_busy_slash_command below — no per-command if-chain here. + if _should_evict: + logger.warning( + "Evicting stale _running_agents entry for %s " + "(age: %.0fs, idle: %.0fs, timeout: %.0fs)%s", + _quick_key, _stale_age, _stale_idle, + _raw_stale_timeout, _stale_detail, + ) + self._invalidate_session_run_generation(_quick_key, reason="stale_running_agent_eviction") + self._release_running_agent_state(_quick_key) + + def _hm_evict_reaped_agent(self, _quick_key: str) -> None: + """Evict the in-memory turn slot of a session whose durable row was ended while the gateway lived. + + (``ws_orphan_reap`` / ``agent_close``): otherwise the fast-path queues every next message + into the dead runtime. The cold path then re-attaches via ``get_or_create_session`` → + ``reopen`` or creates a fresh session. + """ + try: + _reap_store = getattr(self, "session_store", None) + # Public, lock-held accessors: peek_session_id resolves key -> session_id under the + # store lock and returns a non-str on stubbed stores in bare test runners — the + # isinstance() and ``is True`` gates keep this inert unless a real SessionStore answers. + _reap_peek = getattr(_reap_store, "peek_session_id", None) + _is_ended = getattr(_reap_store, "_is_session_ended_in_db", None) + _reap_sid = _reap_peek(_quick_key) if callable(_reap_peek) else None + if ( + isinstance(_reap_sid, str) + and _reap_sid + and callable(_is_ended) + and _is_ended(_reap_sid) is True + ): + logger.warning( + "Evicting stale _running_agents entry for %s — " + "durable session %s is ended (reaped) in state.db; " + "healing routing on next message (#99106)", + _quick_key, _reap_sid, + ) + self._invalidate_session_run_generation(_quick_key, reason="reaped_session_eviction") + self._release_running_agent_state(_quick_key) + except Exception: + logger.debug("reaped-session staleness check failed", exc_info=True) + + def _hm_evict_stale_running_agent(self, _quick_key: str) -> None: + """Evict a leaked/reaped ``_running_agents`` slot before the busy-session fast-path.""" + self._hm_evict_idle_stale_agent(_quick_key) + if self._is_session_running(_quick_key): + self._hm_evict_reaped_agent(_quick_key) + + def _hm_merge_pending_for_source( + self, source: SessionSource, _quick_key: str, event: "MessageEvent", *, merge_text: bool = False + ) -> None: + """Merge *event* into the source adapter's pending slot (no-op without an adapter).""" + from gateway.run import merge_pending_message_event + adapter = self._adapter_for_source(source) + if adapter: + merge_pending_message_event(adapter._pending_messages, _quick_key, event, merge_text=merge_text) + + async def _hm_busy_slash_or_photo( + self, event: "MessageEvent", source: SessionSource, _quick_key: str + ) -> Tuple[bool, Optional[str]]: + """Slash-command / photo-burst handling on the busy fast-path → ``(handled, result)``. Each + command's mid-run behavior is declared on its CommandDef (busy_policy / busy_handler).""" from hermes_cli.commands import resolve_command as _resolve_cmd_inner _evt_cmd = event.get_command() _cmd_def_inner = _resolve_cmd_inner(_evt_cmd) if _evt_cmd else None - # /status and /context are intentionally pre-gate so users - # always see session state. + # /status and /context are intentionally pre-gate so users always see session state. if _cmd_def_inner and _cmd_def_inner.name == "status": - return await self._handle_status_command(event) + return True, await self._handle_status_command(event) if _cmd_def_inner and _cmd_def_inner.name == "context": - return await self._handle_context_command(event) + return True, await self._handle_context_command(event) - # Slash command access control on the running-agent fast-path. Mirrors the cold-path - # gate below so non-admins can't bypass gating just because an agent is busy. /status - # above is intentionally pre-gate; /help and /whoami are the always-allowed floor. + # Slash access control mirrors the cold-path gate so non-admins can't bypass gating just + # because an agent is busy. /help and /whoami are the always-allowed floor. if _evt_cmd and _cmd_def_inner is not None: _denied = self._check_slash_access(source, _cmd_def_inner.name) if _denied is not None: - return _denied + return True, _denied - # Any recognized slash command: dispatch according to its declared busy_policy (dispatch - # / interrupt_then_dispatch / reject). Unrecognized commands and plain text fall through - # to the interrupt/queue logic below. + # Any recognized slash command dispatches per its declared busy_policy (dispatch / + # interrupt_then_dispatch / reject). Unrecognized commands and plain text fall through. if _cmd_def_inner: - return await self._dispatch_busy_slash_command( - event, _cmd_def_inner, _quick_key, source, - ) + return True, await self._dispatch_busy_slash_command(event, _cmd_def_inner, _quick_key, source) + # Telegram photo bursts arrive as near-simultaneous updates — never interrupt for a + # photo-only follow-up; adapter-level batching absorbs them. if event.message_type == MessageType.PHOTO: logger.debug("PRIORITY photo follow-up for session %s — queueing without interrupt", _quick_key) - adapter = self._adapter_for_source(source) - if adapter: - merge_pending_message_event(adapter._pending_messages, _quick_key, event) - return None + self._hm_merge_pending_for_source(source, _quick_key, event) + return True, None + return False, None - effective_busy_input_mode = self._effective_busy_input_mode(source) - _telegram_followup_grace = float( - os.getenv("HERMES_TELEGRAM_FOLLOWUP_GRACE_SECONDS", "3.0") - ) + def _hm_busy_telegram_grace_queue( + self, event: "MessageEvent", source: SessionSource, _quick_key: str, effective_busy_input_mode: str + ) -> bool: + """Queue a Telegram text follow-up that lands within the post-start grace window.""" + _telegram_followup_grace = float(os.getenv("HERMES_TELEGRAM_FOLLOWUP_GRACE_SECONDS", "3.0")) _grace_state = self._peek_session_state(_quick_key) _started_at = _grace_state.turn.started_ts if _grace_state else 0 - if ( + if not ( source.platform == Platform.TELEGRAM and event.message_type == MessageType.TEXT and _telegram_followup_grace > 0 and _started_at and (time.time() - _started_at) <= _telegram_followup_grace ): - logger.debug( - "Telegram follow-up arrived %.2fs after run start for %s — queueing without interrupt", - time.time() - _started_at, - _quick_key, + return False + logger.debug( + "Telegram follow-up arrived %.2fs after run start for %s — queueing without interrupt", + time.time() - _started_at, _quick_key, + ) + adapter = self._adapter_for_source(source) + if adapter: + if effective_busy_input_mode == "queue": + self._enqueue_fifo(_quick_key, event, adapter) + else: + self._hm_merge_pending_for_source(source, _quick_key, event, merge_text=True) + return True + + def _hm_busy_steer(self, event: "MessageEvent", running_agent: Any, _quick_key: str) -> None: + """Steer mode: inject text mid-run via ``agent.steer()``, else fall back to queue semantics.""" + steer_text = (event.text or "").strip() + steered = False + if ( + event.message_type == MessageType.TEXT + and not event.media_urls + and not event.media_types + and steer_text + and hasattr(running_agent, "steer") + ): + try: + steered = bool(running_agent.steer(steer_text)) + except Exception as exc: + logger.warning("PRIORITY steer failed for session %s: %s", _quick_key, exc) + steered = False + if steered: + logger.debug("PRIORITY steer for session %s", _quick_key) + return + logger.debug("PRIORITY steer-fallback-to-queue for session %s", _quick_key) + self._queue_or_replace_pending_event(_quick_key, event) + + async def _hm_busy_interrupt( + self, event: "MessageEvent", source: SessionSource, running_agent: Any, _quick_key: str + ) -> None: + """Interrupt path: redirect text-only corrections when supported, else ``agent.interrupt()``.""" + from gateway.run import _build_media_placeholder + # Text-only corrections redirect the live turn (preserving displayed context) when the + # runtime supports it; media/voice and older runtimes use the interrupt path below. + if ( + event.message_type == MessageType.TEXT + and not event.media_urls + and not event.media_types + and getattr(running_agent, "_supports_active_turn_redirect", False) is True + and hasattr(running_agent, "redirect") + ): + try: + if running_agent.redirect((event.text or "").strip()): + logger.debug("PRIORITY redirect for session %s", _quick_key) + return + except Exception as exc: + logger.warning("PRIORITY redirect failed for session %s: %s", _quick_key, exc) + logger.debug("PRIORITY interrupt for session %s", _quick_key) + _interrupt_text = event.text + _media_urls = getattr(event, "media_urls", None) or [] + if self._pending_event_audio_paths(event): + _interrupt_text, _ = await self._transcribe_and_echo_pending_voice( + event, self._adapter_for_source(source), source, event.text or "", + log_context="Voice-priority-interrupt", ) - adapter = self._adapter_for_source(source) - if adapter: - if effective_busy_input_mode == "queue": - self._enqueue_fifo(_quick_key, event, adapter) - else: - merge_pending_message_event( - adapter._pending_messages, - _quick_key, - event, - merge_text=True, - ) + elif not _interrupt_text and _media_urls: + _interrupt_text = _build_media_placeholder(event) + running_agent.interrupt(_interrupt_text) + # The interrupt message is delivered via adapter._pending_messages (read by _run_agent); + # don't also buffer it on self — that copy was never consumed and grew unbounded. + + async def _hm_handle_running_session_message( + self, event: "MessageEvent", source: SessionSource, _quick_key: str + ) -> Optional[str]: + """Fast-path while this session's agent is running: interrupt by default (minimal latency); + busy_input_mode queue/steer, subagent and compression protection demote to queue.""" + from gateway.run import _AGENT_PENDING_SENTINEL + _handled, _result = await self._hm_busy_slash_or_photo(event, source, _quick_key) + if _handled: + return _result + + effective_busy_input_mode = self._effective_busy_input_mode(source) + if self._hm_busy_telegram_grace_queue(event, source, _quick_key, effective_busy_input_mode): return None _ra_state = self._peek_session_state(_quick_key) @@ -687,21 +703,11 @@ class GatewayInboundMixin: self._release_running_agent_state(_quick_key) logger.info("HARD STOP (pending) for session %s — sentinel cleared", _quick_key) return EphemeralReply("⚡ Force-stopped. The agent was still starting — session unlocked.") - # Queue the message so it will be picked up after the - # agent starts. - adapter = self._adapter_for_source(source) - if adapter: - merge_pending_message_event( - adapter._pending_messages, - _quick_key, - event, - merge_text=True, - ) + # Queue the message so it will be picked up after the agent starts. + self._hm_merge_pending_for_source(source, _quick_key, event, merge_text=True) return None if self._draining: - queue_during_drain = self._queue_during_drain_enabled( - effective_busy_input_mode - ) + queue_during_drain = self._queue_during_drain_enabled(effective_busy_input_mode) if queue_during_drain: self._queue_or_replace_pending_event(_quick_key, event) return ( @@ -714,33 +720,11 @@ class GatewayInboundMixin: self._queue_or_replace_pending_event(_quick_key, event) return None if effective_busy_input_mode == "steer": - # Steer mode: inject text into the running agent mid-run via - # agent.steer(). Falls back to queue semantics if the payload - # is empty, the agent lacks steer(), or steer() rejects. - steer_text = (event.text or "").strip() - steered = False - if ( - event.message_type == MessageType.TEXT - and not event.media_urls - and not event.media_types - and steer_text - and hasattr(running_agent, "steer") - ): - try: - steered = bool(running_agent.steer(steer_text)) - except Exception as exc: - logger.warning("PRIORITY steer failed for session %s: %s", _quick_key, exc) - steered = False - if steered: - logger.debug("PRIORITY steer for session %s", _quick_key) - return None - logger.debug("PRIORITY steer-fallback-to-queue for session %s", _quick_key) - self._queue_or_replace_pending_event(_quick_key, event) + self._hm_busy_steer(event, running_agent, _quick_key) return None - # Subagent protection (PRIORITY path). Same rationale as - # ``_handle_active_session_busy_message``: an interrupt cascades through - # ``_active_children`` and aborts in-flight delegate_task work, so demote to queue - # semantics while subagents run. /stop reached its handler above — still an escape hatch. + # Subagent protection: an interrupt cascades through ``_active_children`` and aborts + # in-flight delegate_task work, so demote to queue while subagents run. /stop reached its + # handler above — still an escape hatch. if self._agent_has_active_subagents(running_agent): logger.info( "PRIORITY interrupt demoted to queue for session %s " @@ -749,9 +733,8 @@ class GatewayInboundMixin: ) self._queue_or_replace_pending_event(_quick_key, event) return None - # Compression protection (PRIORITY path), as in ``_handle_active_session_busy_message``: - # an interrupt would start a new turn on the pre-rotation parent while compression - # rotates the id away, forking orphaned siblings. Demote to queue until rotation lands. + # Compression protection: an interrupt would start a new turn on the pre-rotation parent + # while compression rotates the id away, forking orphaned siblings. if await self._session_has_compression_in_flight(_quick_key): logger.info( "PRIORITY interrupt demoted to queue for session %s " @@ -760,364 +743,349 @@ class GatewayInboundMixin: ) self._queue_or_replace_pending_event(_quick_key, event) return None - # Text-only corrections redirect the live turn (preserving displayed context) when the - # runtime supports it; media/voice and older runtimes use the interrupt path below. - if ( - event.message_type == MessageType.TEXT - and not event.media_urls - and not event.media_types - and getattr(running_agent, "_supports_active_turn_redirect", False) - is True - and hasattr(running_agent, "redirect") - ): - try: - if running_agent.redirect((event.text or "").strip()): - logger.debug("PRIORITY redirect for session %s", _quick_key) - return None - except Exception as exc: - logger.warning( - "PRIORITY redirect failed for session %s: %s", - _quick_key, - exc, - ) - logger.debug("PRIORITY interrupt for session %s", _quick_key) - _interrupt_text = event.text - _media_urls = getattr(event, "media_urls", None) or [] - if self._pending_event_audio_paths(event): - _interrupt_text, _ = await self._transcribe_and_echo_pending_voice( - event, - self._adapter_for_source(source), - source, - event.text or "", - log_context="Voice-priority-interrupt", - ) - elif not _interrupt_text and _media_urls: - _interrupt_text = _build_media_placeholder(event) - running_agent.interrupt(_interrupt_text) - # The interrupt message is delivered via adapter._pending_messages (read by _run_agent); - # don't also buffer it on self — that copy was never consumed and grew unbounded. + await self._hm_busy_interrupt(event, source, running_agent, _quick_key) return None + def _hm_quick_commands(self) -> dict: + """User-defined ``quick_commands`` mapping from config (empty dict when unset/malformed).""" + if isinstance(self.config, dict): + quick_commands = self.config.get("quick_commands", {}) or {} + else: + quick_commands = getattr(self.config, "quick_commands", {}) or {} + return quick_commands if isinstance(quick_commands, dict) else {} + + @staticmethod + def _hm_expand_alias_quick_command(event: "MessageEvent", qcmd: dict) -> Optional[str]: + """Rewrite ``event.text`` to an alias quick command's target; returns the new command name.""" + target = (qcmd.get("target") or "").strip() + if not target: + return None + target = target if target.startswith("/") else f"/{target}" + target_command = target.lstrip("/") + user_args = event.get_command_args().strip() + event.text = f"{target} {user_args}".strip() + return target_command.split()[0] if target_command else target_command + + async def _hm_command_hooks( + self, event: "MessageEvent", source: SessionSource, _quick_key: str, command: str, canonical: str + ) -> Tuple[bool, Optional[str], Optional[str]]: + """Fire ``pre_command`` (observer) and ``command:`` (interceptor) hooks. + + Returns ``(handled, result, new_command)``; ``new_command`` is set when a handler rewrote + the command. The running-agent intercept path deliberately does NOT fire these — a slow or + hostile plugin must not interfere with the operator's escape hatches for a live agent. + """ + try: + from hermes_cli.plugins import fire_pre_command_hook + fire_pre_command_hook( + surface="gateway", command=str(canonical), alias_used=str(command), + args_raw=event.get_command_args().strip(), session_key=_quick_key, + platform=source.platform.value if source.platform else "", + ) + except Exception as _pre_cmd_err: + logger.debug("pre_command hook dispatch failed (non-fatal): %s", _pre_cmd_err) + + # Handlers may return ``{"decision": "deny" | "handled" | "rewrite", ...}`` to intercept + # dispatch; handlers returning nothing behave as plain observers. + raw_args = event.get_command_args().strip() + hook_ctx = { + "platform": source.platform.value if source.platform else "", + "user_id": source.user_id, + "command": canonical, + "raw_command": command, + "args": raw_args, + "raw_args": raw_args, + } + try: + hook_results = await self.hooks.emit_collect(f"command:{canonical}", hook_ctx) + except Exception as _hook_err: + logger.debug("command:%s hook dispatch failed (non-fatal): %s", canonical, _hook_err) + hook_results = [] + + for hook_result in hook_results: + if not isinstance(hook_result, dict): + continue + decision = str(hook_result.get("decision", "")).strip().lower() + if not decision or decision == "allow": + continue + message = hook_result.get("message") + message = message if isinstance(message, str) and message else None + if decision == "deny": + return True, message or f"Command `/{command}` was blocked by a hook.", None + if decision == "handled": + return True, message, None + if decision == "rewrite": + new_command = str(hook_result.get("command_name", "")).strip().lstrip("/") + if not new_command: + continue + new_args = str(hook_result.get("raw_args", "")).strip() + event.text = f"/{new_command} {new_args}".strip() + return False, None, event.get_command() + return False, None, None + async def _hm_resolve_command( self, event: "MessageEvent", source: SessionSource, _quick_key: str ) -> Tuple[bool, Optional[str], Optional[str], Optional[str]]: - """Resolve the slash command (aliases, access gate, ``pre_command`` + ``command:`` hooks). - - Returns ``(handled, result, command, canonical)``; when ``handled`` the caller returns - ``result`` as-is (it may legitimately be None). - """ - # Check for commands + """Resolve the slash command (aliases, access gate, hooks) → ``(handled, result, command, + canonical)``; when ``handled`` the caller returns ``result`` as-is (may be None).""" command = event.get_command() - from hermes_cli.commands import ( - is_gateway_known_command, - resolve_command as _resolve_cmd, - ) + from hermes_cli.commands import is_gateway_known_command, resolve_command as _resolve_cmd - # Resolve aliases to canonical name so dispatch and hook names - # don't depend on the exact alias the user typed. + # Resolve aliases to canonical name so dispatch and hook names don't depend on the alias. _cmd_def = _resolve_cmd(command) if command else None canonical = _cmd_def.name if _cmd_def else command # Expand alias quick commands before built-in dispatch so targets like /model openai/gpt-5.5 - # --provider openrouter reach the /model handler. Preserve built-in precedence; aliases only + # --provider openrouter reach the /model handler. Built-ins keep precedence: aliases only # need early handling when the typed command is not already known. if command and _cmd_def is None: - if isinstance(self.config, dict): - quick_commands = self.config.get("quick_commands", {}) or {} - else: - quick_commands = getattr(self.config, "quick_commands", {}) or {} - if isinstance(quick_commands, dict) and command in quick_commands: - qcmd = quick_commands[command] - if qcmd.get("type") == "alias": - target = (qcmd.get("target") or "").strip() - if target: - target = target if target.startswith("/") else f"/{target}" - target_command = target.lstrip("/") - user_args = event.get_command_args().strip() - event.text = f"{target} {user_args}".strip() - command = target_command.split()[0] if target_command else target_command - _cmd_def = _resolve_cmd(command) if command else None - canonical = _cmd_def.name if _cmd_def else command - - # Per-platform slash command access control. Only kicks in when the operator has set - # ``allow_admin_from`` for the source's scope (DM vs group). When unset → backward-compat: - # every allowed user can run every command. When set → non-admins get only - # ``user_allowed_commands`` plus the /help, /whoami floor. Plain chat is never gated. - if command and canonical and is_gateway_known_command(canonical): - _denied = self._check_slash_access(source, canonical) - if _denied is not None: - return True, _denied, command, canonical - - # pre_command observer hook (returns ignored) fires for every recognized slash command - # BEFORE core handling, mirroring cli.py. The running-agent intercept path above (/stop, - # /approve, busy_policy) deliberately does NOT fire it — a slow or hostile plugin must not - # interfere with the operator's escape hatches for a live agent. - if command and is_gateway_known_command(canonical): - try: - from hermes_cli.plugins import fire_pre_command_hook - fire_pre_command_hook( - surface="gateway", - command=str(canonical), - alias_used=str(command), - args_raw=event.get_command_args().strip(), - session_key=_quick_key, - platform=source.platform.value if source.platform else "", - ) - except Exception as _pre_cmd_err: - logger.debug( - "pre_command hook dispatch failed (non-fatal): %s", - _pre_cmd_err, - ) - - # Fire ``command:`` for any recognized slash command (built-in or plugin). - # Handlers may return ``{"decision": "deny" | "handled" | "rewrite", ...}`` to intercept - # dispatch; handlers returning nothing behave as plain observers. - if command and is_gateway_known_command(canonical): - raw_args = event.get_command_args().strip() - hook_ctx = { - "platform": source.platform.value if source.platform else "", - "user_id": source.user_id, - "command": canonical, - "raw_command": command, - "args": raw_args, - "raw_args": raw_args, - } - try: - hook_results = await self.hooks.emit_collect( - f"command:{canonical}", hook_ctx - ) - except Exception as _hook_err: - logger.debug( - "command:%s hook dispatch failed (non-fatal): %s", - canonical, _hook_err, - ) - hook_results = [] - - for hook_result in hook_results: - if not isinstance(hook_result, dict): - continue - decision = str(hook_result.get("decision", "")).strip().lower() - if not decision or decision == "allow": - continue - if decision == "deny": - message = hook_result.get("message") - if isinstance(message, str) and message: - return True, message, command, canonical - return True, f"Command `/{command}` was blocked by a hook.", command, canonical - if decision == "handled": - message = hook_result.get("message") - return True, message if isinstance(message, str) and message else None, command, canonical - if decision == "rewrite": - new_command = str( - hook_result.get("command_name", "") - ).strip().lstrip("/") - if not new_command: - continue - new_args = str(hook_result.get("raw_args", "")).strip() - event.text = f"/{new_command} {new_args}".strip() - command = event.get_command() + quick_commands = self._hm_quick_commands() + qcmd = quick_commands[command] if command in quick_commands else None + if qcmd is not None and qcmd.get("type") == "alias": + new_command = self._hm_expand_alias_quick_command(event, qcmd) + if new_command is not None: + command = new_command _cmd_def = _resolve_cmd(command) if command else None canonical = _cmd_def.name if _cmd_def else command - break + if not (command and canonical and is_gateway_known_command(canonical)): + return False, None, command, canonical + + # Per-platform slash access control: only active when the operator set ``allow_admin_from`` + # for the source's scope; then non-admins get ``user_allowed_commands`` plus the + # /help, /whoami floor. Plain chat is never gated. + _denied = self._check_slash_access(source, canonical) + if _denied is not None: + return True, _denied, command, canonical + + _handled, _result, new_command = await self._hm_command_hooks( + event, source, _quick_key, command, canonical + ) + if _handled: + return True, _result, command, canonical + if new_command is not None: + command = new_command + _cmd_def = _resolve_cmd(command) if command else None + canonical = _cmd_def.name if _cmd_def else command return False, None, command, canonical + async def _hm_cmd_new(self, event, source, _quick_key): + if await asyncio.to_thread(self._is_telegram_topic_root_lobby, source): + return True, self._telegram_topic_root_new_message() + async def _do_reset(): + return await self._handle_reset_command(event) + return True, await self._maybe_confirm_destructive_slash( + event=event, command="new", title="/new", + detail="This starts a fresh session and discards the current conversation history.", + execute=_do_reset, + ) + + async def _hm_cmd_start(self, event, source, _quick_key): + logger.info("Ignoring /start platform ping for session %s", _quick_key) + return True, "" + + async def _hm_cmd_egress(self, event, source, _quick_key): + from hermes_cli.proxy_cli import format_status_text + + return True, format_status_text() + + async def _hm_rewrite_turn_to_prompt(self, event, source, name: str, ack: str, build) -> Tuple[bool, Optional[str]]: + """Ack, then rewrite the turn to ``build()`` and fall through to the agent (keeps role + alternation; works on any backend). A failing builder replies with a retry hint.""" + await self._send_command_ack(source, ack, name) + try: + event.text = build() + except Exception: + return True, f"Could not start /{name} — please try again." + return False, None + + async def _hm_cmd_learn(self, event, source, _quick_key): + from agent.learn_prompt import build_learn_prompt + + _learn_req = event.get_command_args().strip() + _ack = ( + "Learning a skill from what you described…" + if _learn_req + else "Learning a skill from this conversation…" + ) + return await self._hm_rewrite_turn_to_prompt( + event, source, "learn", _ack, lambda: build_learn_prompt(_learn_req) + ) + + async def _hm_cmd_plan(self, event, source, _quick_key): + from agent.plan_prompt import build_plan_prompt + + _plan_task = event.get_command_args().strip() + _ack = ( + f"Planning: {_plan_task[:80]}{'…' if len(_plan_task) > 80 else ''}" + if _plan_task + else "Planning from this conversation's context…" + ) + return await self._hm_rewrite_turn_to_prompt( + event, source, "plan", _ack, lambda: build_plan_prompt(_plan_task) + ) + + async def _hm_cmd_init(self, event, source, _quick_key): + # /init builds the prompt first: the ack wording depends on whether AGENTS.md exists. + from hermes_cli.init_command import build_init_prompt_for_cwd + + _init_notes = event.get_command_args().strip() + try: + _init_prompt = build_init_prompt_for_cwd(extra=_init_notes) + except Exception: + return True, "Could not start /init — please try again." + _ack = ( + "Updating AGENTS.md from a project scan…" + if "UPDATE the existing AGENTS.md" in _init_prompt + else "Generating AGENTS.md from a project scan…" + ) + await self._send_command_ack(source, _ack, "init") + event.text = _init_prompt + return False, None + + async def _hm_cmd_blueprint(self, event, source, _quick_key): + _blueprint_result = await self._handle_blueprint_command(event) + _blueprint_seed = getattr(_blueprint_result, "agent_seed", None) + if not _blueprint_seed: + return True, getattr(_blueprint_result, "text", "") or None + # Blueprint matched — rewrite the turn to the seed and fall through so the agent collects + # each slot value conversationally, then calls the cronjob tool (the /steer pattern). + _ack = getattr(_blueprint_result, "text", "") or "" + if _ack: + await self._send_command_ack(source, _ack, "blueprint") + try: + event.text = _blueprint_seed + except Exception: + return True, getattr(_blueprint_result, "text", "") or None + return False, None + + async def _hm_cmd_undo(self, event, source, _quick_key): + async def _do_undo(): + return await self._handle_undo_command(event) + _undo_n = 1 + _undo_raw = event.get_command_args().strip() + if _undo_raw: + try: + _undo_n = max(1, int(_undo_raw.split()[0])) + except (ValueError, IndexError): + _undo_n = 1 + _undo_detail = ( + "This removes the last user/assistant exchange from history." + if _undo_n == 1 + else f"This removes the last {_undo_n} user turns from history." + ) + return True, await self._maybe_confirm_destructive_slash( + event=event, command="undo", title="/undo", detail=_undo_detail, execute=_do_undo, + ) + + async def _hm_cmd_queue(self, event, source, _quick_key): + queue_payload = event.get_command_args().strip() + if not queue_payload: + return True, "Usage: /queue " + with suppress(Exception): + event.text = queue_payload + return False, None + + async def _hm_cmd_steer(self, event, source, _quick_key): + # No active agent — /steer has nothing to inject into. Strip the prefix so the rewritten + # text is sent to the agent as a regular user turn; an empty payload surfaces the usage hint. + steer_payload = event.get_command_args().strip() + if not steer_payload: + return True, "Usage: /steer (no agent is running; sending as a normal message)" + with suppress(Exception): + event.text = steer_payload + return False, None + + async def _hm_cmd_moa(self, event, source, _quick_key): + # /moa is one-shot sugar only: run a single prompt through the default MoA preset, then + # restore the prior model. To *switch* to a MoA preset for the session, pick it from the + # model picker (MoA presets surface as a virtual "Mixture of Agents" provider). + from hermes_cli.moa_config import moa_usage, normalize_moa_config + from hermes_cli.config import load_config + + moa_payload = event.get_command_args().strip() + if not moa_payload: + return True, moa_usage() + try: + cfg = load_config() + moa_cfg = normalize_moa_config(cfg.get("moa") if isinstance(cfg, dict) else {}) + except Exception: + moa_cfg = normalize_moa_config({}) + preset = moa_cfg["default_preset"] + try: + event.text = moa_payload + _moa_state = self._session_state(_quick_key) + event._moa_restore_override = _moa_state.conversation.model_override + _moa_state.conversation.model_override = { + "provider": "moa", + "model": preset, + "base_url": "moa://local", + "api_key": "moa-virtual-provider", + "api_mode": "chat_completions", + } + self._evict_cached_agent(_quick_key) + event._moa_disable_after_turn = True + except Exception: + return True, "Failed to prepare MoA turn." + return False, None + + # Idle-path built-ins with bespoke flow (confirmations, prompt rewrites, one-shot MoA). Each + # returns ``(handled, result)``; ``(False, None)`` falls through to the agent. + _HM_CANONICAL_COMMANDS = { + "new": "_hm_cmd_new", + "start": "_hm_cmd_start", + "egress": "_hm_cmd_egress", + "learn": "_hm_cmd_learn", + "plan": "_hm_cmd_plan", + "init": "_hm_cmd_init", + "blueprint": "_hm_cmd_blueprint", + "undo": "_hm_cmd_undo", + "queue": "_hm_cmd_queue", + "steer": "_hm_cmd_steer", + "moa": "_hm_cmd_moa", + } + async def _hm_dispatch_canonical_command( - self, - event: "MessageEvent", - source: SessionSource, - _quick_key: str, + self, event: "MessageEvent", source: SessionSource, _quick_key: str, canonical: Optional[str], ) -> Tuple[bool, Optional[str]]: - """Dispatch built-in idle-path commands (plain handlers, /new, /learn, /plan, /moa, ...). - - Returns ``(handled, result)``. Prompt-rewriting commands (/learn, /plan, /init, /steer, - /moa, ...) mutate ``event.text`` and return ``(False, None)`` to fall through to the agent. - """ + """Dispatch built-in idle-path commands → ``(handled, result)``; prompt-rewriting commands + mutate ``event.text`` and return ``(False, None)`` to fall through to the agent.""" plain_handler = ( self._gateway_plain_command_handlers().get(canonical) or self._gateway_idle_command_handlers().get(canonical) ) if plain_handler is not None: return True, await plain_handler(event) - - if canonical == "new": - if await asyncio.to_thread(self._is_telegram_topic_root_lobby, source): - return True, self._telegram_topic_root_new_message() - async def _do_reset(): - return await self._handle_reset_command(event) - return True, await self._maybe_confirm_destructive_slash( - event=event, - command="new", - title="/new", - detail=( - "This starts a fresh session and discards the current " - "conversation history." - ), - execute=_do_reset, - ) - - if canonical == "start": - logger.info("Ignoring /start platform ping for session %s", _quick_key) - return True, "" - - if canonical == "egress": - from hermes_cli.proxy_cli import format_status_text - - return True, format_status_text() - - if canonical == "learn": - # Open-ended: rewrite the turn to a standards-guided prompt and fall through to normal - # agent processing. Mirrors the /blueprint fall-through so role alternation is - # preserved. No engine, works on any backend. - from agent.learn_prompt import build_learn_prompt - - _learn_req = event.get_command_args().strip() - _ack = ( - "Learning a skill from what you described…" - if _learn_req - else "Learning a skill from this conversation…" - ) - await self._send_command_ack(source, _ack, "learn") - try: - event.text = build_learn_prompt(_learn_req) - # fall through to agent processing - except Exception: - return True, "Could not start /learn — please try again." - - if canonical == "plan": - # /plan: rewrite the turn to the plan-mode prompt and fall through to normal agent - # processing (the /learn fall-through keeps role alternation). Works on any backend. - from agent.plan_prompt import build_plan_prompt - - _plan_task = event.get_command_args().strip() - _ack = ( - f"Planning: {_plan_task[:80]}{'…' if len(_plan_task) > 80 else ''}" - if _plan_task - else "Planning from this conversation's context…" - ) - await self._send_command_ack(source, _ack, "plan") - try: - event.text = build_plan_prompt(_plan_task) - # fall through to agent processing - except Exception: - return True, "Could not start /plan — please try again." - - if canonical == "init": - # /init: rewrite the turn to a guidance-laden prompt and fall through to normal agent - # processing (the /learn fall-through keeps role alternation). Works on any backend. - from hermes_cli.init_command import build_init_prompt_for_cwd - - _init_notes = event.get_command_args().strip() - try: - _init_prompt = build_init_prompt_for_cwd(extra=_init_notes) - except Exception: - return True, "Could not start /init — please try again." - _ack = ( - "Updating AGENTS.md from a project scan…" - if "UPDATE the existing AGENTS.md" in _init_prompt - else "Generating AGENTS.md from a project scan…" - ) - await self._send_command_ack(source, _ack, "init") - event.text = _init_prompt - # fall through to agent processing - - if canonical == "blueprint": - _blueprint_result = await self._handle_blueprint_command(event) - _blueprint_seed = getattr(_blueprint_result, "agent_seed", None) - if _blueprint_seed: - # Blueprint matched — rewrite the turn to the seed and fall through to - # _handle_message_with_agent so the agent collects each slot value conversationally, - # then calls the cronjob tool (the /steer fall-through pattern). - _ack = getattr(_blueprint_result, "text", "") or "" - if _ack: - await self._send_command_ack(source, _ack, "blueprint") - try: - event.text = _blueprint_seed - except Exception: - return True, getattr(_blueprint_result, "text", "") or None - else: - return True, getattr(_blueprint_result, "text", "") or None - - if canonical == "undo": - async def _do_undo(): - return await self._handle_undo_command(event) - _undo_n = 1 - _undo_raw = event.get_command_args().strip() - if _undo_raw: - try: - _undo_n = max(1, int(_undo_raw.split()[0])) - except (ValueError, IndexError): - _undo_n = 1 - _undo_detail = ( - "This removes the last user/assistant exchange from history." - if _undo_n == 1 - else f"This removes the last {_undo_n} user turns from history." - ) - return True, await self._maybe_confirm_destructive_slash( - event=event, - command="undo", - title="/undo", - detail=_undo_detail, - execute=_do_undo, - ) - - if canonical == "queue": - queue_payload = event.get_command_args().strip() - if not queue_payload: - return True, "Usage: /queue " - with suppress(Exception): - event.text = queue_payload - - if canonical == "steer": - # No active agent — /steer has nothing to inject into. Strip the prefix so downstream - # treats it as a normal user message; an empty payload surfaces the usage hint. - steer_payload = event.get_command_args().strip() - if not steer_payload: - return True, "Usage: /steer (no agent is running; sending as a normal message)" - with suppress(Exception): - event.text = steer_payload - # Do NOT return — fall through to _handle_message_with_agent at the end of this function - # so the rewritten text is sent to the agent as a regular user turn. - - if canonical == "moa": - # /moa is one-shot sugar only: run a single prompt through the default MoA preset, then - # restore the prior model. To *switch* to a MoA preset for the session, pick it from the - # model picker (MoA presets surface as a virtual "Mixture of Agents" provider). - from hermes_cli.moa_config import ( - moa_usage, - normalize_moa_config, - ) - from hermes_cli.config import load_config - - moa_payload = event.get_command_args().strip() - if not moa_payload: - return True, moa_usage() - try: - cfg = load_config() - moa_cfg = normalize_moa_config(cfg.get("moa") if isinstance(cfg, dict) else {}) - except Exception: - moa_cfg = normalize_moa_config({}) - preset = moa_cfg["default_preset"] - try: - event.text = moa_payload - _moa_state = self._session_state(_quick_key) - event._moa_restore_override = _moa_state.conversation.model_override - _moa_state.conversation.model_override = { - "provider": "moa", - "model": preset, - "base_url": "moa://local", - "api_key": "moa-virtual-provider", - "api_mode": "chat_completions", - } - self._evict_cached_agent(_quick_key) - event._moa_disable_after_turn = True - except Exception: - return True, "Failed to prepare MoA turn." - + bespoke = self._HM_CANONICAL_COMMANDS.get(canonical) + if bespoke is not None: + return await getattr(self, bespoke)(event, source, _quick_key) return False, None + async def _hm_run_exec_quick_command(self, command: str, exec_cmd: str) -> str: + """Run a ``type: exec`` quick command in the gateway process (30 s cap, sanitized env).""" + try: + # Sanitize env to prevent credential leakage — the gateway process has all API keys + # in os.environ. + from tools.environments.local import build_subprocess_env + sanitized_env = build_subprocess_env() + proc = await asyncio.create_subprocess_shell( + exec_cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, + env=sanitized_env, + ) + stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=30) + output = (stdout or stderr).decode().strip() + # Redact any remaining sensitive patterns in output + if output: + from agent.redact import redact_sensitive_text + output = redact_sensitive_text(output) + return output if output else "Command returned no output." + except asyncio.TimeoutError: + return "Quick command timed out (30s)." + except Exception as e: + return f"Quick command error: {e}" + async def _hm_dispatch_quick_and_plugin_commands( self, event: "MessageEvent", source: SessionSource, command: Optional[str] ) -> Tuple[bool, Optional[str], Optional[str]]: @@ -1129,62 +1097,28 @@ class GatewayInboundMixin: return True, f"⏳ Gateway is {self._status_action_gerund()} and is not accepting new work right now.", command # User-defined quick commands (bypass agent loop, no LLM call) - if command: - if isinstance(self.config, dict): - quick_commands = self.config.get("quick_commands", {}) or {} + quick_commands = self._hm_quick_commands() + if command and command in quick_commands: + qcmd = quick_commands[command] + # Quick commands are slash capabilities too — and type:exec ones run a shell command in + # the gateway process. They are never in the registry, so the early gate never fires for + # them; apply the same admin/user policy to the raw typed name here. + _denied = self._check_slash_access(source, command) + if _denied is not None: + return True, _denied, command + qtype = qcmd.get("type") + if qtype == "exec": + exec_cmd = qcmd.get("command", "") + if not exec_cmd: + return True, f"Quick command '/{command}' has no command defined.", command + return True, await self._hm_run_exec_quick_command(command, exec_cmd), command + if qtype == "alias": + new_command = self._hm_expand_alias_quick_command(event, qcmd) + if new_command is None: + return True, f"Quick command '/{command}' has no target defined.", command + command = new_command # Fall through to normal command dispatch below else: - quick_commands = getattr(self.config, "quick_commands", {}) or {} - if not isinstance(quick_commands, dict): - quick_commands = {} - if command in quick_commands: - # Quick commands are slash capabilities too — and type:exec ones run a shell command - # in the gateway process. The early gate only fires for registry-known commands and - # quick commands are never in the registry, so apply the same admin/user policy to - # the raw typed name here so non-admins can't invoke admin-only quick commands. - _denied = self._check_slash_access(source, command) - if _denied is not None: - return True, _denied, command - qcmd = quick_commands[command] - if qcmd.get("type") == "exec": - exec_cmd = qcmd.get("command", "") - if exec_cmd: - try: - # Sanitize env to prevent credential leakage — quick commands run in the - # gateway process, which has all API keys in os.environ. - from tools.environments.local import build_subprocess_env - sanitized_env = build_subprocess_env() - proc = await asyncio.create_subprocess_shell( - exec_cmd, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - env=sanitized_env, - ) - stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=30) - output = (stdout or stderr).decode().strip() - # Redact any remaining sensitive patterns in output - if output: - from agent.redact import redact_sensitive_text - output = redact_sensitive_text(output) - return True, output if output else "Command returned no output.", command - except asyncio.TimeoutError: - return True, "Quick command timed out (30s).", command - except Exception as e: - return True, f"Quick command error: {e}", command - else: - return True, f"Quick command '/{command}' has no command defined.", command - elif qcmd.get("type") == "alias": - target = (qcmd.get("target") or "").strip() - if target: - target = target if target.startswith("/") else f"/{target}" - target_command = target.lstrip("/") - user_args = event.get_command_args().strip() - event.text = f"{target} {user_args}".strip() - command = target_command.split()[0] if target_command else target_command - # Fall through to normal command dispatch below - else: - return True, f"Quick command '/{command}' has no target defined.", command - else: - return True, f"Quick command '/{command}' has unsupported type (supported: 'exec', 'alias').", command + return True, f"Quick command '/{command}' has unsupported type (supported: 'exec', 'alias').", command # Plugin-registered slash commands if command: @@ -1204,157 +1138,203 @@ class GatewayInboundMixin: return False, None, command + def _hm_bundle_slash_rewrite( + self, event: "MessageEvent", source: SessionSource, _quick_key: str, command: str + ) -> bool: + """Rewrite ``/`` to the bundle invocation message; True when handled. + + Skill bundles take precedence over individual skill commands (mirrors CLI dispatch). + """ + try: + from agent.skill_bundles import ( + build_bundle_invocation_message, resolve_bundle_command_key + ) + bundle_key = resolve_bundle_command_key(command) + if bundle_key is None: + return False + user_instruction = event.get_command_args().strip() + # Pass the platform explicitly: bundle skill loading bypasses get_skill_commands()' + # scan-time disabled filter, and one gateway process serves several platforms, so + # env-var platform resolution can't be trusted here. + _bundle_plat = source.platform.value if source.platform else None + bundle_result = build_bundle_invocation_message( + bundle_key, user_instruction, task_id=_quick_key, platform=_bundle_plat, + ) + if not bundle_result: + return False + msg, _loaded, missing = bundle_result + event.text = msg + if missing: + logger.info("Bundle %s skipped missing skills: %s", bundle_key, ", ".join(missing)) + return True # Fall through to normal message processing with bundle content + except Exception as exc: + logger.warning("Bundle dispatch failed: %s", exc) + return False + + @staticmethod + def _hm_unknown_slash_reply(command: str, source: SessionSource) -> Optional[str]: + """Reply for a /command that is not built-in/plugin/skill; None when it is known.""" + from gateway.run import _check_unavailable_skill + from hermes_cli.commands import GATEWAY_KNOWN_COMMANDS + # Known-but-disabled or uninstalled skill → actionable guidance. + _unavail_msg = _check_unavailable_skill(command) + if _unavail_msg: + return _unavail_msg + # Genuinely unrecognized: warn instead of forwarding to the LLM as free text (it invents + # tool calls). Normalize to hyphenated form first: the quick-command block may have set an + # alias target, so the resolved def can be stale. + if command.replace("_", "-") in GATEWAY_KNOWN_COMMANDS: + return None + logger.warning( + "Unrecognized slash command /%s from %s — replying with unknown-command notice", + command, source.platform.value if source.platform else "?", + ) + return ( + f"Unknown command `/{command}`. " + f"Type /commands to see what's available, " + f"or resend without the leading slash to send " + f"as a regular message." + ) + def _hm_skill_slash_rewrite( - self, - event: "MessageEvent", - source: SessionSource, - _quick_key: str, - command: Optional[str], + self, event: "MessageEvent", source: SessionSource, _quick_key: str, command: Optional[str] ) -> Optional[str]: """Rewrite ``/`` / ``/`` invocations into the skill prompt on ``event.text``. Returns a reply string when the command is disabled/unknown/failed, else None. + resolve_skill_command_key() handles the Telegram underscore/hyphen round-trip so + /claude_code from Telegram autocomplete still resolves to the claude-code skill. """ - from gateway.run import _check_unavailable_skill - from hermes_cli.commands import GATEWAY_KNOWN_COMMANDS - - # Skill slash commands: /skill-name loads the skill and sends to agent. - # resolve_skill_command_key() handles the Telegram underscore/hyphen round-trip so - # /claude_code from Telegram autocomplete still resolves to the claude-code skill. - if command: - # Skill bundles take precedence over individual skill commands — - # / loads multiple skills at once. Mirrors CLI dispatch. - _bundle_handled = False - try: - from agent.skill_bundles import ( - build_bundle_invocation_message, - resolve_bundle_command_key, - ) - bundle_key = resolve_bundle_command_key(command) - if bundle_key is not None: - user_instruction = event.get_command_args().strip() - # Pass the platform explicitly: bundle skill loading bypasses - # get_skill_commands()' scan-time disabled filter, and the gateway serves - # multiple platforms in one process, so env-var platform resolution can't be - # trusted here. - _bundle_plat = source.platform.value if source.platform else None - bundle_result = build_bundle_invocation_message( - bundle_key, user_instruction, task_id=_quick_key, - platform=_bundle_plat, + if not command or self._hm_bundle_slash_rewrite(event, source, _quick_key, command): + return None + try: + from agent.skill_commands import ( + get_skill_commands, build_skill_invocation_message, resolve_skill_command_key + ) + skill_cmds = get_skill_commands() + cmd_key = resolve_skill_command_key(command) + if cmd_key is None: + return self._hm_unknown_slash_reply(command, source) + # Per-platform disabled check: get_skill_commands() only applies the *global* disabled + # list at scan time and its cache is process-global across platforms. + _skill_name = skill_cmds[cmd_key].get("name", "") + _plat = source.platform.value if source.platform else None + if _plat and _skill_name: + from agent.skill_utils import get_disabled_skill_names as _get_plat_disabled + if _skill_name in _get_plat_disabled(platform=_plat): + return ( + f"The **{_skill_name}** skill is disabled for {_plat}.\n" + f"Enable it with: `hermes skills config`" ) - if bundle_result: - msg, _loaded, missing = bundle_result - event.text = msg - _bundle_handled = True - if missing: - logger.info( - "Bundle %s skipped missing skills: %s", - bundle_key, ", ".join(missing), - ) - # Fall through to normal message processing with bundle content - except Exception as exc: - logger.warning("Bundle dispatch failed: %s", exc) - - if command and not locals().get("_bundle_handled", False): + user_instruction = event.get_command_args().strip() + # Stacked slash-skill invocations: `/skill-a /skill-b do XYZ` loads every leading skill + # (up to 5), not just the first. Mirrors CLI. try: from agent.skill_commands import ( - get_skill_commands, - build_skill_invocation_message, - resolve_skill_command_key, + build_stacked_skill_invocation_message as _build_stacked, + split_stacked_skill_commands, ) - skill_cmds = get_skill_commands() - cmd_key = resolve_skill_command_key(command) - if cmd_key is not None: - # Check per-platform disabled status before executing. get_skill_commands() only - # applies the *global* disabled list at scan time; per-platform overrides need - # checking here because the cache is process-global across platforms. - _skill_name = skill_cmds[cmd_key].get("name", "") - _plat = source.platform.value if source.platform else None - if _plat and _skill_name: - from agent.skill_utils import get_disabled_skill_names as _get_plat_disabled - if _skill_name in _get_plat_disabled(platform=_plat): - return ( - f"The **{_skill_name}** skill is disabled for {_plat}.\n" - f"Enable it with: `hermes skills config`" - ) - user_instruction = event.get_command_args().strip() - # Stacked slash-skill invocations: `/skill-a /skill-b do XYZ` loads every - # leading skill (up to 5), not just the first. Mirrors CLI. - try: - from agent.skill_commands import ( - build_stacked_skill_invocation_message as _build_stacked, - split_stacked_skill_commands, - ) - extra_keys, stacked_instruction = ( - split_stacked_skill_commands(user_instruction) - ) - except Exception: - _build_stacked = None - extra_keys, stacked_instruction = [], user_instruction - if extra_keys and _plat: - # split_stacked_skill_commands() only resolves that each extra token is a - # KNOWN skill command — like get_skill_commands() itself, it has no per- - # platform view. Re-check every stacked skill against the same disabled list, - # or a skill disabled for this platform still gets loaded via the stack. - from agent.skill_utils import get_disabled_skill_names as _get_plat_disabled - _plat_disabled = _get_plat_disabled(platform=_plat) - _disabled_extra = [ - skill_cmds.get(k, {}).get("name", "") - for k in extra_keys - if skill_cmds.get(k, {}).get("name", "") in _plat_disabled - ] - if _disabled_extra: - return ( - f"The **{', '.join(_disabled_extra)}** skill(s) in this " - f"stacked invocation are disabled for {_plat}.\n" - f"Enable them with: `hermes skills config`" - ) - if extra_keys and _build_stacked is not None: - stacked_result = _build_stacked( - [cmd_key, *extra_keys], - stacked_instruction, - task_id=_quick_key, - ) - if stacked_result: - msg, _loaded, _missing = stacked_result - event.text = msg - # Fall through to normal message processing - else: - return f"Failed to load stacked skills for /{command}." - else: - msg = build_skill_invocation_message( - cmd_key, user_instruction, task_id=_quick_key - ) - if msg: - event.text = msg - # Fall through to normal message processing with skill content - else: - # Not an active skill — check if it's a known-but-disabled or - # uninstalled skill and give actionable guidance. - _unavail_msg = _check_unavailable_skill(command) - if _unavail_msg: - return _unavail_msg - # Genuinely unrecognized /command (not built-in/plugin/skill/known-inactive): - # warn instead of forwarding to the LLM as free text (it invents tool calls). - # Normalize to hyphenated form first: the quick-command block may have set an - # alias target, so _cmd_def can be stale. - if command.replace("_", "-") not in GATEWAY_KNOWN_COMMANDS: - logger.warning( - "Unrecognized slash command /%s from %s — " - "replying with unknown-command notice", - command, - source.platform.value if source.platform else "?", - ) - return ( - f"Unknown command `/{command}`. " - f"Type /commands to see what's available, " - f"or resend without the leading slash to send " - f"as a regular message." - ) - except Exception as e: - logger.debug("Skill command check failed (non-fatal): %s", e) + extra_keys, stacked_instruction = split_stacked_skill_commands(user_instruction) + except Exception: + _build_stacked = None + extra_keys, stacked_instruction = [], user_instruction + if extra_keys and _plat: + # split_stacked_skill_commands() only checks each extra token is a KNOWN skill; it + # has no per-platform view, so re-check every stacked skill against the disabled list. + from agent.skill_utils import get_disabled_skill_names as _get_plat_disabled + _plat_disabled = _get_plat_disabled(platform=_plat) + _disabled_extra = [ + skill_cmds.get(k, {}).get("name", "") + for k in extra_keys + if skill_cmds.get(k, {}).get("name", "") in _plat_disabled + ] + if _disabled_extra: + return ( + f"The **{', '.join(_disabled_extra)}** skill(s) in this " + f"stacked invocation are disabled for {_plat}.\n" + f"Enable them with: `hermes skills config`" + ) + if extra_keys and _build_stacked is not None: + stacked_result = _build_stacked( + [cmd_key, *extra_keys], stacked_instruction, task_id=_quick_key, + ) + if not stacked_result: + return f"Failed to load stacked skills for /{command}." + msg, _loaded, _missing = stacked_result + event.text = msg + else: + msg = build_skill_invocation_message(cmd_key, user_instruction, task_id=_quick_key) + if msg: + event.text = msg + # Fall through to normal message processing with skill content + except Exception as e: + logger.debug("Skill command check failed (non-fatal): %s", e) return None + async def _hm_pending_reply_intercepts( + self, event: "MessageEvent", source: SessionSource, _quick_key: str + ) -> Optional[str]: + """Replies owned by in-flight work: pending /update prompt, clarify, slash-confirm.""" + allow_gateway_control = event.allow_gateway_control + _reply = self._hm_update_prompt_reply(event, _quick_key, allow_gateway_control) + if _reply is not None: + return _reply + _reply = await self._hm_clarify_reply(event, source, _quick_key, allow_gateway_control) + if _reply is not None: + return _reply + return await self._hm_slash_confirm_reply(event, _quick_key, allow_gateway_control) + + async def _hm_dispatch_idle_commands( + self, event: "MessageEvent", source: SessionSource, _quick_key: str + ) -> Tuple[bool, Optional[str]]: + """Idle path: resolve + dispatch slash commands; rewriting commands fall through to the agent.""" + _handled, _result, command, canonical = await self._hm_resolve_command(event, source, _quick_key) + if _handled: + return True, _result + _handled, _result = await self._hm_dispatch_canonical_command(event, source, _quick_key, canonical) + if _handled: + return True, _result + _handled, _result, command = await self._hm_dispatch_quick_and_plugin_commands(event, source, command) + if _handled: + return True, _result + _skill_reply = self._hm_skill_slash_rewrite(event, source, _quick_key, command) + if _skill_reply is not None: + return True, _skill_reply + return False, None + + def _hm_rescue_orphaned_fifo( + self, event: "MessageEvent", source: SessionSource, is_internal: bool, _quick_key: str + ) -> Tuple["MessageEvent", SessionSource, bool]: + """FIFO orphan rescue: re-stage overflow events left behind by an idle session. + + A session that went idle with a populated overflow (post-turn drain never promoted, e.g. a + compression-demoted follow-up) silently orphaned those events. The oldest orphan runs as + THIS turn and the incoming event is parked behind the chain. Skipped for control commands + and internal events. + """ + try: + _orphan_adapter = self._adapter_for_source(source) + if ( + _orphan_adapter is not None + and not bool(getattr(event, "internal", False)) + and not event.get_command() + ): + _rescued = self._rescue_orphaned_overflow(_quick_key, _orphan_adapter) + if _rescued is not None: + # Into the slot when the chain was a single orphan (post-turn drain picks it + # up), otherwise into overflow behind the already-staged next orphan. + self._enqueue_fifo(_quick_key, event, _orphan_adapter) + event = _rescued + # Same session key by construction; carry the orphan's own source so reply + # anchors / thread metadata point at the message actually being answered. + _rescued_source = getattr(_rescued, "source", None) + if _rescued_source is not None: + source = _rescued_source + is_internal = bool(getattr(_rescued, "internal", False)) + except Exception: + logger.debug("FIFO orphan rescue pre-claim failed for %s", _quick_key, exc_info=True) + return event, source, is_internal + async def _handle_message(self, event: MessageEvent) -> Optional[str]: """Handle an incoming message from any platform. @@ -1371,121 +1351,49 @@ class GatewayInboundMixin: if _paused_notice is not None: return _paused_notice - # Replies owned by in-flight work: pending /update prompt, clarify, slash-confirm. _quick_key = self._session_key_for_source(source) - allow_gateway_control = event.allow_gateway_control - _update_reply = self._hm_update_prompt_reply(event, _quick_key, allow_gateway_control) - if _update_reply is not None: - return _update_reply - _clarify_reply = await self._hm_clarify_reply( - event, source, _quick_key, allow_gateway_control - ) - if _clarify_reply is not None: - return _clarify_reply - _confirm_reply = await self._hm_slash_confirm_reply( - event, _quick_key, allow_gateway_control - ) - if _confirm_reply is not None: - return _confirm_reply + _reply = await self._hm_pending_reply_intercepts(event, source, _quick_key) + if _reply is not None: + return _reply self._hm_evict_stale_running_agent(_quick_key) if self._is_session_running(_quick_key): return await self._hm_handle_running_session_message(event, source, _quick_key) - # Idle path: resolve + dispatch slash commands; rewriting commands fall through to the agent. - _handled, _result, command, canonical = await self._hm_resolve_command( - event, source, _quick_key - ) + _handled, _result = await self._hm_dispatch_idle_commands(event, source, _quick_key) if _handled: return _result - _handled, _result = await self._hm_dispatch_canonical_command( - event, source, _quick_key, canonical - ) - if _handled: - return _result - _handled, _result, command = await self._hm_dispatch_quick_and_plugin_commands( - event, source, command - ) - if _handled: - return _result - _skill_reply = self._hm_skill_slash_rewrite(event, source, _quick_key, command) - if _skill_reply is not None: - return _skill_reply # Pending exec approvals go through /approve and /deny only — no bare-text matching, or a # conversational "yes" would execute a dangerous command. - if not is_internal and await asyncio.to_thread( - self._is_telegram_topic_root_lobby, source - ): - # Debounce the lobby reminder so a user who forgets about - # topic mode and fires ten prompts doesn't get ten copies. + if not is_internal and await asyncio.to_thread(self._is_telegram_topic_root_lobby, source): + # Debounce the lobby reminder so a user who forgets about topic mode and fires ten + # prompts doesn't get ten copies. if self._should_send_telegram_lobby_reminder(source): return self._telegram_topic_root_lobby_message() return None - # ── External-drain new-turn gate ───────────────────────────── - # When NAS engaged an external drain (.drain_request.json, seen by _drain_control_watcher), - # refuse to START new turns so the in-flight set can only fall to zero (stop accepting - # FIRST, then NAS polls active_agents==0). Internal/system events bypass; reversible. + # External-drain new-turn gate: when NAS engaged an external drain (.drain_request.json, + # seen by _drain_control_watcher), refuse to START new turns so the in-flight set can only + # fall to zero. Internal/system events bypass; reversible. if self._external_drain_active and not is_internal: - logger.info( - "Refusing new turn for session %s — external drain active.", - _quick_key, - ) + logger.info("Refusing new turn for session %s — external drain active.", _quick_key) return ( "⏳ This agent is draining for a maintenance action and isn't " "accepting new turns right now. It'll be back in a moment — " "please resend shortly." ) - # ── Claim this session before any await ─────────────────────── - # Many awaits sit between here and _run_agent registering the real AIAgent; without this - # sentinel a second message during any of them passes the "already running" guard and spins - # up a duplicate agent for the same session, corrupting the transcript. - _active_session_lease, _limit_message = self._claim_active_session_slot( - _quick_key, - source, - ) + # Claim this session before any await: many awaits sit between here and _run_agent + # registering the real AIAgent; without this sentinel a second message during any of them + # passes the "already running" guard and spins up a duplicate agent for the same session. + _active_session_lease, _limit_message = self._claim_active_session_slot(_quick_key, source) if _limit_message is not None: - logger.info( - "Rejecting new active session %s: max_concurrent_sessions reached", - _quick_key, - ) + logger.info("Rejecting new active session %s: max_concurrent_sessions reached", _quick_key) return _limit_message - # ── FIFO orphan rescue ─────────────────────────────────────── - # A session that went idle with a populated overflow (post-turn drain never promoted, e.g. a - # compression-demoted follow-up) silently orphaned those events. Re-stage them FIFO and - # enqueue this event behind them. Skipped for control commands and internal events. - try: - _orphan_adapter = self._adapter_for_source(source) - if ( - _orphan_adapter is not None - and not bool(getattr(event, "internal", False)) - and not event.get_command() - ): - _rescued = self._rescue_orphaned_overflow( - _quick_key, _orphan_adapter - ) - if _rescued is not None: - # The oldest orphan runs as THIS turn. Park the incoming event behind the rest of - # the chain: into the slot when the chain was a single orphan (post-turn drain - # picks it up), otherwise into overflow behind the already-staged next orphan. - self._enqueue_fifo(_quick_key, event, _orphan_adapter) - event = _rescued - # Same session key by construction; carry the orphan's own source so reply - # anchors / thread metadata point at the message actually being answered. - _rescued_source = getattr(_rescued, "source", None) - if _rescued_source is not None: - source = _rescued_source - is_internal = bool(getattr(_rescued, "internal", False)) - except Exception: - logger.debug( - "FIFO orphan rescue pre-claim failed for %s", - _quick_key, - exc_info=True, - ) + event, source, is_internal = self._hm_rescue_orphaned_fifo(event, source, is_internal, _quick_key) _claim_state = self._session_state(_quick_key) if _active_session_lease is not None: @@ -1501,14 +1409,13 @@ class GatewayInboundMixin: event, source, _quick_key, _run_generation ) except TurnLeaseTimeoutError as exc: - # A rejected message, not a completed turn: return before the /goal judge below so - # it cannot consume the resend notice and enqueue a synthetic continuation loop. + # A rejected message, not a completed turn: return before the /goal judge so it + # cannot consume the resend notice and enqueue a synthetic continuation loop. logger.error( "Rejecting turn for routing key %s on session %s after " "turn-lease timeout; transcript load was not started and " "the user must resend", - _quick_key, - exc.session_id, + _quick_key, exc.session_id, ) return ( "⏳ Another turn is still running on this session. To " @@ -1517,30 +1424,25 @@ class GatewayInboundMixin: ) try: await self._run_post_turn_hooks( - agent_result=_agent_result, - source=source, - is_internal=is_internal, - event=event, + agent_result=_agent_result, source=source, is_internal=is_internal, event=event, ) except Exception as _goal_exc: logger.debug("post-turn hook failed: %s", _goal_exc) return _agent_result finally: - # MoA one-shot restore must run on EVERY exit path: the restore data lives on the - # per-turn event, so a restore in the try block is skipped when the handler raises and - # the override leaks permanently; finally covers success, exception and interrupt. + # MoA one-shot restore must run on EVERY exit path (success, exception, interrupt): + # the restore data lives on the per-turn event and would leak permanently otherwise. self._restore_moa_one_shot(event, _quick_key) self._restore_pending_one_turn_model_override(_quick_key) - # Normal completion/exception/interrupt clears this durable marker; SIGKILL/OOM skips - # finally, leaving it for the next unclean startup's recovery pass. + # SIGKILL/OOM skips finally, leaving the durable marker for the next unclean startup's + # recovery pass. await self._clear_durable_active_turn(event) - # Unconditional release covers every exit path: _release_running_agent_state is idempotent - # and, without a run_generation guard, clears the slot whichever generation holds it. This - # evicts the zombie left when session_reset bumps the generation mid-flight (gen-N's - # guarded release in _run_agent returns False; a sentinel-only check would lock forever). + # Unconditional, idempotent release without a run_generation guard: evicts the zombie + # left when session_reset bumps the generation mid-flight (gen-N's guarded release in + # _run_agent returns False; a sentinel-only check would lock forever). self._release_running_agent_state(_quick_key) - # Turn lease: release THIS turn's token — keyed by (routing key, run generation) so this - # unwind can only free the lease its own turn acquired, never a newer turn's. + # Turn lease is keyed by (routing key, run generation) so this unwind can only free + # the lease its own turn acquired, never a newer turn's. self._release_turn_lease(_quick_key, _run_generation) def _restore_moa_one_shot(self, event: "MessageEvent", quick_key: str) -> None: @@ -1576,31 +1478,23 @@ class GatewayInboundMixin: def _prefix_inbound_sender_context(self, event: MessageEvent, source: SessionSource, message_text: str) -> str: """Attribute the sender in shared multi-user sessions and prepend history-backfill channel context.""" - _group_sessions_per_user = getattr(self.config, "group_sessions_per_user", True) - _thread_sessions_per_user = getattr(self.config, "thread_sessions_per_user", False) _is_shared_multi_user = is_shared_multi_user_session( - source, - group_sessions_per_user=_group_sessions_per_user, - thread_sessions_per_user=_thread_sessions_per_user, + source, group_sessions_per_user=getattr(self.config, "group_sessions_per_user", True), + thread_sessions_per_user=getattr(self.config, "thread_sessions_per_user", False), ) if _is_shared_multi_user and source.user_name: - # source.user_name is the platform display name — attacker-influenceable on any - # platform that lets participants set their own name. Neutralize newlines/control chars - # before interpolating it into every message, or a hostile name can masquerade as a - # fake markdown section (mirrors build_session_context_prompt's treatment). + # The display name is attacker-influenceable on platforms where participants set their + # own name: neutralize newlines/control chars or a hostile name can masquerade as a fake + # markdown section (mirrors build_session_context_prompt's treatment). _safe_user_name = neutralize_untrusted_inline_text(source.user_name) - # On Slack, expose the current author's verifiable user ID next to the display name: - # "mention me again" requests need a trusted `<@U...>` target for the CURRENT speaker — - # display names are ambiguous and historical mentions may point at someone else. The - # user_id comes from the Slack event envelope (not user-editable), so no neutralization. + # Slack: expose the current author's verifiable user ID so "mention me again" requests + # have a trusted `<@U...>` target for the CURRENT speaker (display names are ambiguous). + # The user_id comes from the event envelope (not user-editable), so no neutralization. if source.platform == Platform.SLACK and source.user_id: - _safe_user_name = ( - f"{_safe_user_name} | Slack user <@{source.user_id}>" - ) + _safe_user_name = f"{_safe_user_name} | Slack user <@{source.user_id}>" message_text = f"[{_safe_user_name}] {message_text}" - # Prepend history-backfill channel context after the sender-prefix so the prefix applies - # only to the trigger message, not the backfill block. + # After the sender-prefix so the prefix applies only to the trigger message, not the backfill. if getattr(event, "channel_context", None): message_text = f"{event.channel_context}\n\n[New message]\n{message_text}" return message_text @@ -1609,218 +1503,171 @@ class GatewayInboundMixin: def _classify_inbound_media( event: MessageEvent, pending_stt_prepared: bool ) -> Tuple[list, list, list, list]: - """Split ``event.media_urls`` into (image, STT-voice, audio-file, video) paths.""" + """Split ``event.media_urls`` into (image, STT-voice, audio-file, video) paths. + + Per-attachment MIME wins over the message-level type (a document sent alongside an image + must not be routed as an image). MessageType.AUDIO / mixed DOCUMENT audio is a file + attachment, never STT. + """ from gateway.run import _event_media_is_audio, _event_media_is_image, _event_media_is_stt_input image_paths: list[str] = [] audio_paths: list[str] = [] audio_file_paths: list[str] = [] video_paths: list[str] = [] - - if event.media_urls: - for i, path in enumerate(event.media_urls): - mtype = event.media_types[i] if i < len(event.media_types) else "" - # Classify images per-attachment: trust this attachment's own MIME, and only honour - # the message-level PHOTO type when the per-attachment MIME is unknown. Otherwise a - # document sent alongside an image gets mis-routed as an image and the provider 400s. - if _event_media_is_image(event, i): - image_paths.append(path) - # MessageType.AUDIO = audio file attachment (e.g. .mp3, .m4a) — never STT. - # Mixed DOCUMENT events also preserve audio as a file path instead of - # dropping it or treating it as a voice note. - if _event_media_is_audio(event, i): - if event.message_type in {MessageType.AUDIO, MessageType.DOCUMENT}: - audio_file_paths.append(path) - elif not pending_stt_prepared and _event_media_is_stt_input(event, i): - audio_paths.append(path) - if mtype.startswith("video/") or (not mtype and event.message_type == MessageType.VIDEO): - video_paths.append(path) + for i, path in enumerate(event.media_urls or []): + mtype = event.media_types[i] if i < len(event.media_types) else "" + if _event_media_is_image(event, i): + image_paths.append(path) + if _event_media_is_audio(event, i): + if event.message_type in {MessageType.AUDIO, MessageType.DOCUMENT}: + audio_file_paths.append(path) + elif not pending_stt_prepared and _event_media_is_stt_input(event, i): + audio_paths.append(path) + if mtype.startswith("video/") or (not mtype and event.message_type == MessageType.VIDEO): + video_paths.append(path) return image_paths, audio_paths, audio_file_paths, video_paths async def _enrich_inbound_images( self, source: SessionSource, session_key: str, message_text: str, image_paths: list[str] ) -> str: - # Decide routing: native (attach pixels) vs text (vision_analyze pre-run + prepend - # description). See agent/image_routing.py. Offloaded to a thread: the decision does - # blocking network I/O (models.dev fetch on cache miss, Ollama /api/show probe) whose - # timeout would otherwise stall the whole gateway event loop. + """Route images natively (attach pixels at run_conversation) or pre-analyze them into text.""" + # See agent/image_routing.py. Offloaded to a thread: the decision does blocking network I/O + # (models.dev fetch on cache miss, Ollama /api/show probe) that would stall the event loop. _img_mode = await asyncio.to_thread( - self._decide_image_input_mode, - source=source, - session_key=session_key, + self._decide_image_input_mode, source=source, session_key=session_key, ) if _img_mode == "native": - # Defer attachment to the run_conversation call site. - self._session_state( - session_key - ).persistent.native_image_paths = list(image_paths) + self._session_state(session_key).persistent.native_image_paths = list(image_paths) logger.info( "Image routing: native (model supports vision). %d image(s) will be attached inline.", len(image_paths), ) - else: - logger.info( - "Image routing: text (mode=%s). Pre-analyzing %d image(s) via vision_analyze.", - _img_mode, len(image_paths), + return message_text + logger.info( + "Image routing: text (mode=%s). Pre-analyzing %d image(s) via vision_analyze.", + _img_mode, len(image_paths), + ) + # Vision enrichment runs before AIAgent.run_conversation(), so bind this session's resolved + # runtime explicitly rather than consulting process-global compatibility mirrors. + vision_runtime = None + try: + turn_model, runtime_kwargs = self._resolve_session_agent_runtime( + source=source, session_key=session_key, ) - # Vision enrichment runs before AIAgent.run_conversation(), - # so bind this session's resolved runtime explicitly rather - # than consulting process-global compatibility mirrors. - vision_runtime = None + vision_runtime = dict(runtime_kwargs or {}) + vision_runtime["model"] = turn_model + except Exception: + logger.debug("vision enrichment: session runtime resolution failed", exc_info=True) + + from agent.auxiliary_client import scoped_runtime_main + + with scoped_runtime_main(vision_runtime): + return await self._enrich_message_with_vision(message_text, image_paths) + + async def _echo_stt_transcripts( + self, adapter, source: SessionSource, transcripts: List[str], *, metadata=None, log_context: str = "Transcript" + ) -> None: + """Send each transcript back as ``🎙️ "…"`` (best-effort; failures are logged, never raised).""" + for tx in transcripts: try: - turn_model, runtime_kwargs = self._resolve_session_agent_runtime( - source=source, - session_key=session_key, - ) - vision_runtime = dict(runtime_kwargs or {}) - vision_runtime["model"] = turn_model - except Exception: - logger.debug( - "vision enrichment: session runtime resolution failed", - exc_info=True, - ) - - from agent.auxiliary_client import scoped_runtime_main - - with scoped_runtime_main(vision_runtime): - message_text = await self._enrich_message_with_vision( - message_text, - image_paths, - ) - return message_text + await adapter.send(source.chat_id, f'🎙️ "{tx}"', metadata=metadata) + except Exception as echo_exc: + logger.debug("%s echo failed (non-fatal): %s", log_context, echo_exc) async def _enrich_inbound_voice( self, event: MessageEvent, source: SessionSource, message_text: str, audio_paths: list[str] ) -> str: message_text, _successful_transcripts = await self._enrich_message_with_transcription( - message_text, - audio_paths, + message_text, audio_paths, ) - # Echo each successful transcript back to the user immediately when configured. Lets - # users verify STT quality in real-time, while allowing quiet STT for users who only - # want the agent to receive the transcription. + # Echo each successful transcript back immediately when configured so users can verify STT + # quality in real time (quiet STT stays available for users who only want the agent to + # receive it). On transcription failure do NOT send a hardcoded notice: that bypassed the + # LLM and produced two replies; enrichment leaves one neutral marker for a localized reply. if _successful_transcripts and self._should_echo_stt_transcripts(): _echo_adapter = self._adapter_for_source(source) _echo_meta = self._thread_metadata_for_source(source, self._reply_anchor_for_event(event)) if _echo_adapter: - for _tx in _successful_transcripts: - try: - await _echo_adapter.send( - source.chat_id, - f'🎙️ "{_tx}"', - metadata=_echo_meta, - ) - except Exception as _echo_exc: - logger.debug( - "Transcript echo failed (non-fatal): %s", _echo_exc, - ) - # On transcription failure, do NOT send a hardcoded notice here: that bypassed the - # LLM and produced two replies (one pre-canned, TTS'd in the wrong language). - # Enrichment leaves a single neutral marker so the LLM gives one localized reply. + await self._echo_stt_transcripts(_echo_adapter, source, _successful_transcripts, metadata=_echo_meta) return message_text @staticmethod - def _prepend_inbound_media_file_notes(message_text: str, audio_file_paths: list[str], video_paths: list[str]) -> str: - if audio_file_paths: - from tools.credential_files import to_agent_visible_cache_path as _to_agent_path - for _apath in audio_file_paths: - _basename = os.path.basename(_apath) - _parts = _basename.split("_", 2) - _display = _parts[2] if len(_parts) >= 3 else _basename - _display = re.sub(r'[^\w.\- ]', '_', _display) - _agent_path = _to_agent_path(_apath) - _note = ( - f"[The user sent an audio file attachment: '{_display}'. " - f"It is saved at: {_agent_path}. " - f"Its content is not inlined here. If the user's request involves " - f"what the audio contains, transcribe or process it yourself — for " - f"example by passing the path to a transcription or media tool — " - f"instead of asking the user to describe it. Only ask what to do " - f"with it if their intent is genuinely unclear.]" - ) - message_text = f"{_note}\n\n{message_text}" + def _inbound_attachment_display_name(path: str) -> Tuple[str, str]: + """``(display_name, agent_visible_path)``: cache filename is ``__``; the + path is translated to the in-container mount under a Docker backend.""" + from tools.credential_files import to_agent_visible_cache_path + basename = os.path.basename(path) + parts = basename.split("_", 2) + display_name = parts[2] if len(parts) >= 3 else basename + return re.sub(r'[^\w.\- ]', '_', display_name), to_agent_visible_cache_path(path) - if video_paths: - from tools.credential_files import to_agent_visible_cache_path as _to_agent_path - for _vpath in video_paths: - _basename = os.path.basename(_vpath) - _parts = _basename.split("_", 2) - _display = _parts[2] if len(_parts) >= 3 else _basename - _display = re.sub(r'[^\w.\- ]', '_', _display) - _agent_path = _to_agent_path(_vpath) + @classmethod + def _prepend_inbound_media_file_notes(cls, message_text: str, audio_file_paths: list[str], video_paths: list[str]) -> str: + """Prepend a path-pointing note per audio-file / video attachment (content is not inlined).""" + for kind, noun, verb, tool, paths in ( + ("an audio file attachment", "audio", "transcribe or process", "a transcription or media tool", audio_file_paths), + ("a video attachment", "video", "inspect or process", "a video analysis or media tool", video_paths), + ): + for _path in paths: + _display, _agent_path = cls._inbound_attachment_display_name(_path) _note = ( - f"[The user sent a video attachment: '{_display}'. " + f"[The user sent {kind}: '{_display}'. " f"It is saved at: {_agent_path}. " f"Its content is not inlined here. If the user's request involves " - f"what the video contains, inspect or process it yourself — for " - f"example by passing the path to a video analysis or media tool — " + f"what the {noun} contains, {verb} it yourself — for " + f"example by passing the path to {tool} — " f"instead of asking the user to describe it. Only ask what to do " f"with it if their intent is genuinely unclear.]" ) message_text = f"{_note}\n\n{message_text}" return message_text - @staticmethod - def _prepend_inbound_document_notes(event: MessageEvent, message_text: str) -> str: + @classmethod + def _prepend_inbound_document_notes(cls, event: MessageEvent, message_text: str) -> str: + """Prepend a context note per non-media attachment (anything not routed as image/audio/video).""" from gateway.run import ( - _build_document_context_note, - _event_media_is_audio, - _event_media_is_image, + _build_document_context_note, _event_media_is_audio, _event_media_is_image, _event_media_is_video, ) - if event.media_urls: - import mimetypes as _mimetypes - from tools.credential_files import to_agent_visible_cache_path + if not event.media_urls: + return message_text + import mimetypes as _mimetypes - _TEXT_EXTENSIONS = {".txt", ".md", ".csv", ".log", ".json", ".xml", ".yaml", ".yml", ".toml", ".ini", ".cfg"} - for i, path in enumerate(event.media_urls): - # Per-attachment document handling: skip anything already routed as image/audio/video - # above; only genuine non-media files get a context note. A document mixed into a - # PHOTO/VOICE message (message-level type != DOCUMENT) thus still reaches the agent. - if ( - _event_media_is_image(event, i) - or _event_media_is_audio(event, i) - or _event_media_is_video(event, i) - ): - continue - mtype = event.media_types[i] if i < len(event.media_types) else "" - if mtype in {"", "application/octet-stream"}: - _ext = os.path.splitext(path)[1].lower() - if _ext in _TEXT_EXTENSIONS: - mtype = "text/plain" - else: - guessed, _ = _mimetypes.guess_type(path) - mtype = guessed or "application/octet-stream" - # Any accepted file gets a path-pointing context note — we accept - # all file types now, so a non-text/non-application MIME (font/*, - # model/*, etc.) must still tell the agent the file exists. - - basename = os.path.basename(path) - parts = basename.split("_", 2) - display_name = parts[2] if len(parts) >= 3 else basename - display_name = re.sub(r'[^\w.\- ]', '_', display_name) - - # Translate host cache path to in-container path if running under Docker backend. - # This ensures the agent receives a path it can open inside its sandbox, as the - # cache directories are auto-mounted at /root/.hermes/cache/* by get_cache_directory_mounts(). - agent_path = to_agent_visible_cache_path(path) - - inline_flags = getattr(event, "media_text_inlined", None) or [] - inline_flag = inline_flags[i] if i < len(inline_flags) else None - context_note = _build_document_context_note( - display_name, - agent_path, - mtype, - content_inlined=inline_flag is not False, - ) - message_text = f"{context_note}\n\n{message_text}" + _TEXT_EXTENSIONS = {".txt", ".md", ".csv", ".log", ".json", ".xml", ".yaml", ".yml", ".toml", ".ini", ".cfg"} + for i, path in enumerate(event.media_urls): + # A document mixed into a PHOTO/VOICE message (message-level type != DOCUMENT) still + # reaches the agent; only genuine non-media files get a note. + if ( + _event_media_is_image(event, i) + or _event_media_is_audio(event, i) + or _event_media_is_video(event, i) + ): + continue + mtype = event.media_types[i] if i < len(event.media_types) else "" + if mtype in {"", "application/octet-stream"}: + _ext = os.path.splitext(path)[1].lower() + if _ext in _TEXT_EXTENSIONS: + mtype = "text/plain" + else: + guessed, _ = _mimetypes.guess_type(path) + mtype = guessed or "application/octet-stream" + # Every accepted file gets a note — a non-text/non-application MIME (font/*, model/*) + # must still tell the agent the file exists. + display_name, agent_path = cls._inbound_attachment_display_name(path) + inline_flags = getattr(event, "media_text_inlined", None) or [] + inline_flag = inline_flags[i] if i < len(inline_flags) else None + context_note = _build_document_context_note( + display_name, agent_path, mtype, content_inlined=inline_flag is not False, + ) + message_text = f"{context_note}\n\n{message_text}" return message_text @staticmethod def _prepend_inbound_reply_context(event: MessageEvent, source: SessionSource, message_text: str) -> str: + """Prepend the Discord triggering-message id and the reply-to pointer.""" # Discord: surface the triggering message id per-turn on the user message rather than in the - # cached system prompt. message_id changes every turn, so baking it into - # build_session_context_prompt() would bust the agent-cache signature and rebuild the - # AIAgent every message (destroying prompt caching). + # cached system prompt — message_id changes every turn and would bust the agent-cache + # signature (rebuilding the AIAgent every message destroys prompt caching). if ( source is not None and getattr(source, "platform", None) == Platform.DISCORD @@ -1835,27 +1682,90 @@ class GatewayInboundMixin: ) if getattr(event, "reply_to_text", None) and event.reply_to_message_id: - # Always inject the reply-to pointer — even when the quoted text already appears in - # history. The prefix isn't deduplication, it's disambiguation: it tells the agent - # *which* prior message the user is referencing. Token overhead is minimal. + # Always inject the reply-to pointer even when the quoted text is already in history: + # it's disambiguation (*which* prior message), not deduplication. reply_snippet = event.reply_to_text[:500] if getattr(event, "reply_to_is_own_message", False): - message_text = ( - f'[Replying to your previous message: "{reply_snippet}"]\n\n' - f"{message_text}" - ) + message_text = f'[Replying to your previous message: "{reply_snippet}"]\n\n{message_text}' else: message_text = f'[Replying to: "{reply_snippet}"]\n\n{message_text}' return message_text + async def _inbound_model_context_length(self, source: SessionSource, session_key: str) -> int: + """Context length of the model this turn runs on. A global ``model.context_length`` pin + belongs to the configured model, not a /model or channel override; custom-provider limits win.""" + from gateway.run import _load_gateway_config + from agent.model_metadata import get_model_context_length_async + + _msg_config_ctx = None + _msg_cfg = None + _msg_model_cfg = {} + _msg_custom_providers = [] + try: + _msg_cfg = _load_gateway_config() + _msg_model_cfg = _msg_cfg.get("model", {}) + if isinstance(_msg_model_cfg, dict): + _msg_raw_ctx = _msg_model_cfg.get("context_length") + if _msg_raw_ctx is not None: + _msg_config_ctx = int(_msg_raw_ctx) + try: + from hermes_cli.config import get_compatible_custom_providers + + _msg_custom_providers = get_compatible_custom_providers(_msg_cfg) + except Exception: + _msg_custom_providers = _msg_cfg.get("custom_providers") or [] + except Exception: + pass + # GatewayRunner has no self._model/self._base_url; resolve the session's actual runtime. + _msg_model, _msg_runtime = self._resolve_session_agent_runtime( + source=source, session_key=session_key, user_config=_msg_cfg, + ) + _msg_base_url = _msg_runtime.get("base_url") or "" + _msg_configured_model = ( + _msg_model_cfg.get("default") or _msg_model_cfg.get("model") + if isinstance(_msg_model_cfg, dict) + else _msg_model_cfg + ) + if _msg_model != _msg_configured_model: + _msg_config_ctx = None + if _msg_config_ctx is not None and isinstance(_msg_model_cfg, dict): + try: + from hermes_cli.route_identity import should_clear_context_pin_async + + if await should_clear_context_pin_async( + None, # model match already checked above + None, + _msg_model_cfg.get("base_url"), + _msg_base_url, + _msg_model_cfg.get("provider"), + _msg_runtime.get("provider"), + ): + _msg_config_ctx = None + except Exception: + _msg_config_ctx = None + if _msg_custom_providers and _msg_base_url: + try: + from hermes_cli.config import get_custom_provider_context_length + + _msg_custom_ctx = get_custom_provider_context_length( + model=_msg_model, base_url=_msg_base_url, custom_providers=_msg_custom_providers, + ) + if _msg_custom_ctx: + _msg_config_ctx = _msg_custom_ctx + except Exception: + pass + return await get_model_context_length_async( + _msg_model, base_url=_msg_base_url, api_key=_msg_runtime.get("api_key") or "", + config_context_length=_msg_config_ctx, provider=_msg_runtime.get("provider") or "", + custom_providers=_msg_custom_providers, + ) + async def _expand_inbound_context_references( self, source: SessionSource, session_key: str, message_text: str ) -> Optional[str]: """Expand ``@`` context references; returns None when the injection was refused (user notified).""" - from gateway.run import _load_gateway_config try: from agent.context_references import preprocess_context_references_async - from agent.model_metadata import get_model_context_length_async try: from tools.terminal_scope import terminal_env as _ts_env @@ -1863,85 +1773,9 @@ class GatewayInboundMixin: _msg_cwd = os.environ.get("TERMINAL_CWD", os.path.expanduser("~")) else: _msg_cwd = _ts_env("TERMINAL_CWD", os.path.expanduser("~")) - _msg_config_ctx = None - _msg_cfg = None - _msg_model_cfg = {} - _msg_custom_providers = [] - try: - _msg_cfg = _load_gateway_config() - _msg_model_cfg = _msg_cfg.get("model", {}) - if isinstance(_msg_model_cfg, dict): - _msg_raw_ctx = _msg_model_cfg.get("context_length") - if _msg_raw_ctx is not None: - _msg_config_ctx = int(_msg_raw_ctx) - try: - from hermes_cli.config import get_compatible_custom_providers - - _msg_custom_providers = get_compatible_custom_providers(_msg_cfg) - except Exception: - _msg_custom_providers = _msg_cfg.get("custom_providers") or [] - except Exception: - pass - # Resolve the session's actual model/provider/base_url as the hygiene compression - # block does; GatewayRunner has no self._model/self._base_url (AttributeError, - # silently caught below). - _msg_model, _msg_runtime = self._resolve_session_agent_runtime( - source=source, - session_key=session_key, - user_config=_msg_cfg, - ) - _msg_base_url = _msg_runtime.get("base_url") or "" - # A global model.context_length belongs to the configured - # model, not a session /model or channel override. Prefer a - # matching per-custom-provider model limit when available. - _msg_configured_model = ( - _msg_model_cfg.get("default") or _msg_model_cfg.get("model") - if isinstance(_msg_model_cfg, dict) - else _msg_model_cfg - ) - if _msg_model != _msg_configured_model: - _msg_config_ctx = None - if _msg_config_ctx is not None and isinstance(_msg_model_cfg, dict): - try: - from hermes_cli.route_identity import should_clear_context_pin_async - - if await should_clear_context_pin_async( - None, # model match already checked above - None, - _msg_model_cfg.get("base_url"), - _msg_base_url, - _msg_model_cfg.get("provider"), - _msg_runtime.get("provider"), - ): - _msg_config_ctx = None - except Exception: - _msg_config_ctx = None - if _msg_custom_providers and _msg_base_url: - try: - from hermes_cli.config import get_custom_provider_context_length - - _msg_custom_ctx = get_custom_provider_context_length( - model=_msg_model, - base_url=_msg_base_url, - custom_providers=_msg_custom_providers, - ) - if _msg_custom_ctx: - _msg_config_ctx = _msg_custom_ctx - except Exception: - pass - _msg_ctx_len = await get_model_context_length_async( - _msg_model, - base_url=_msg_base_url, - api_key=_msg_runtime.get("api_key") or "", - config_context_length=_msg_config_ctx, - provider=_msg_runtime.get("provider") or "", - custom_providers=_msg_custom_providers, - ) + _msg_ctx_len = await self._inbound_model_context_length(source, session_key) _ctx_result = await preprocess_context_references_async( - message_text, - cwd=_msg_cwd, - context_length=_msg_ctx_len, - allowed_root=_msg_cwd, + message_text, cwd=_msg_cwd, context_length=_msg_ctx_len, allowed_root=_msg_cwd ) if _ctx_result.blocked: _adapter = self._adapter_for_source(source) @@ -1959,11 +1793,7 @@ class GatewayInboundMixin: return message_text async def _prepare_inbound_message_text( - self, - *, - event: MessageEvent, - source: SessionSource, - history: List[Dict[str, Any]], + self, *, event: MessageEvent, source: SessionSource, history: List[Dict[str, Any]], session_key: Optional[str] = None, ) -> Optional[str]: """Prepare inbound event text for the agent. @@ -2006,44 +1836,25 @@ class GatewayInboundMixin: return message_text async def _prepare_profile_scoped_inbound_message_text( - self, - *, - event: MessageEvent, - source: SessionSource, - history: List[Dict[str, Any]], + self, *, event: MessageEvent, source: SessionSource, history: List[Dict[str, Any]], session_key: Optional[str] = None, ) -> Optional[str]: """Run inbound preprocessing under the routed profile when multiplexed.""" from gateway.run import _async_profile_runtime_scope + kwargs = dict(event=event, source=source, history=history, session_key=session_key) if getattr(getattr(self, "config", None), "multiplex_profiles", False): - async with _async_profile_runtime_scope( - self._resolve_profile_home_for_source(source) - ): - return await self._prepare_inbound_message_text( - event=event, - source=source, - history=history, - session_key=session_key, - ) - return await self._prepare_inbound_message_text( - event=event, - source=source, - history=history, - session_key=session_key, - ) + async with _async_profile_runtime_scope(self._resolve_profile_home_for_source(source)): + return await self._prepare_inbound_message_text(**kwargs) + return await self._prepare_inbound_message_text(**kwargs) async def _prepare_clarify_reply_text(self, event) -> str: """Return raw text or successful voice transcripts for a clarify reply.""" if not self._pending_event_audio_paths(event): return (event.text or "").strip() - _, successful_transcripts = await self._transcribe_pending_audio_event_once( - event, "", - ) + _, successful_transcripts = await self._transcribe_pending_audio_event_once(event, "") return "\n\n".join( - transcript.strip() - for transcript in successful_transcripts - if transcript.strip() + transcript.strip() for transcript in successful_transcripts if transcript.strip() ) def _consume_pending_native_image_paths(self, session_key: str) -> List[str]: @@ -2054,20 +1865,12 @@ class GatewayInboundMixin: state.persistent.native_image_paths = [] return paths - async def _mark_durable_active_turn( - self, - event: "MessageEvent", - session_key: str, - ) -> bool: + async def _mark_durable_active_turn(self, event: "MessageEvent", session_key: str) -> bool: """Persist the exact resolved routing key for this running turn.""" try: token = await self.async_session_store.mark_turn_active(session_key) except Exception as exc: - logger.warning( - "Could not persist active-turn marker for %s: %s", - session_key, - exc, - ) + logger.warning("Could not persist active-turn marker for %s: %s", session_key, exc) return False if not token: return False @@ -2078,7 +1881,8 @@ class GatewayInboundMixin: return True async def _clear_durable_active_turn(self, event: "MessageEvent") -> bool: - """Best-effort CAS clear of the marker owned by *event*.""" + """Best-effort CAS clear of the marker owned by *event* (3 attempts; never blocks agent/lease + release — a stale marker is bounded by the agent timeout and clean-start discard).""" session_key = getattr(event, "_gateway_active_turn_session_key", None) token = getattr(event, "_gateway_active_turn_token", None) try: @@ -2087,33 +1891,20 @@ class GatewayInboundMixin: last_error: Optional[Exception] = None for attempt in range(1, 4): try: - return bool( - await self.async_session_store.clear_turn_active( - session_key, token - ) - ) + return bool(await self.async_session_store.clear_turn_active(session_key, token)) except Exception as exc: last_error = exc if attempt < 3: logger.debug( "Retrying active-turn marker cleanup for %s (%d/3): %s", - session_key, - attempt, - exc, + session_key, attempt, exc, ) - # Never let marker cleanup block agent/lease release; a stale marker is bounded by the - # agent timeout and the clean-start orphan-marker discard path. logger.warning( - "Could not clear active-turn marker for %s after 3 attempts: %s", - session_key, - last_error, + "Could not clear active-turn marker for %s after 3 attempts: %s", session_key, last_error, ) return False finally: - for attr in ( - "_gateway_active_turn_session_key", - "_gateway_active_turn_token", - ): + for attr in ("_gateway_active_turn_session_key", "_gateway_active_turn_token"): with suppress(AttributeError): delattr(event, attr) @@ -2122,8 +1913,7 @@ class GatewayInboundMixin: from hermes_cli.plugins import get_plugin_manager get_plugin_manager().set_gateway_message_injector( - self, - self._schedule_plugin_message_injection, + self, self._schedule_plugin_message_injection ) def _clear_plugin_message_injector(self) -> None: @@ -2133,22 +1923,16 @@ class GatewayInboundMixin: get_plugin_manager().clear_gateway_message_injector(self) def _schedule_plugin_message_injection( - self, - *, - session_key: str, - content: str, - plugin_id: str, + self, *, session_key: str, content: str, plugin_id: str ) -> bool: - """Schedule a plugin-triggered turn on the live gateway loop.""" + """Schedule a plugin-triggered turn on the live gateway loop (thread-safe).""" from gateway.run import safe_schedule_threadsafe loop = getattr(self, "_gateway_loop", None) if not getattr(self, "_running", False) or loop is None or loop.is_closed(): return False coro = self._dispatch_plugin_message_injection( - session_key=session_key, - content=content, - plugin_id=plugin_id, + session_key=session_key, content=content, plugin_id=plugin_id, ) try: current_loop = asyncio.get_running_loop() @@ -2160,19 +1944,13 @@ class GatewayInboundMixin: future = loop.create_task(coro) except Exception: coro.close() - logger.warning( - "Plugin message injection scheduling failed", - exc_info=True, - ) + logger.warning("Plugin message injection scheduling failed", exc_info=True) return False self._background_tasks.add(future) future.add_done_callback(self._background_tasks.discard) else: future = safe_schedule_threadsafe( - coro, - loop, - logger=logger, - log_message="Plugin message injection scheduling failed", + coro, loop, logger=logger, log_message="Plugin message injection scheduling failed", log_level=logging.WARNING, ) if future is None: @@ -2186,58 +1964,43 @@ class GatewayInboundMixin: except Exception: logger.warning( "Plugin message injection failed: plugin=%s session=%s", - plugin_id, - session_key, - exc_info=True, + plugin_id, session_key, exc_info=True, ) return if not accepted: logger.warning( - "Plugin message injection was not routed: plugin=%s session=%s", - plugin_id, - session_key, + "Plugin message injection was not routed: plugin=%s session=%s", plugin_id, session_key, ) future.add_done_callback(_log_result) return True async def _dispatch_plugin_message_injection( - self, - *, - session_key: str, - content: str, - plugin_id: str, + self, *, session_key: str, content: str, plugin_id: str ) -> bool: """Route a plugin-triggered turn through the session's live adapter.""" - if not getattr(self, "_running", False) or getattr(self, "_draining", False): - return False + def _accepting() -> bool: + return getattr(self, "_running", False) and not getattr(self, "_draining", False) - entry = await self.async_session_store.lookup_by_session_key(session_key) - if entry is None or entry.origin is None: + if not _accepting(): return False - if not getattr(self, "_running", False) or getattr(self, "_draining", False): + entry = await self.async_session_store.lookup_by_session_key(session_key) + if entry is None or entry.origin is None or not _accepting(): return False source = dataclasses.replace(entry.origin) try: - if not self._is_user_authorized( - source, - allow_adapter_delegation=False, - ): + if not self._is_user_authorized(source, allow_adapter_delegation=False): logger.warning( "Plugin message injection denied by current gateway authorization: " "plugin=%s session=%s", - plugin_id, - session_key, + plugin_id, session_key, ) return False except Exception: logger.warning( - "Plugin message injection authorization check failed: " - "plugin=%s session=%s", - plugin_id, - session_key, - exc_info=True, + "Plugin message injection authorization check failed: plugin=%s session=%s", + plugin_id, session_key, exc_info=True, ) return False @@ -2262,28 +2025,20 @@ class GatewayInboundMixin: await adapter.handle_message(event) logger.info( "Plugin message injection dispatched: plugin=%s session=%s session_id=%s", - plugin_id, - session_key, - entry.session_id, + plugin_id, session_key, entry.session_id, ) return True def _decide_image_input_mode( - self, - *, - source: Optional[SessionSource] = None, - session_key: Optional[str] = None, - user_config: Optional[dict] = None, - provider: Optional[str] = None, + self, *, source: Optional[SessionSource] = None, session_key: Optional[str] = None, + user_config: Optional[dict] = None, provider: Optional[str] = None, model: Optional[str] = None, ) -> str: - """Resolve image-input routing for the effective model this turn. + """Resolve image-input routing (``"native"`` / ``"text"``) for the effective model this turn. - Returns ``"native"`` (attach pixels on the user turn) or ``"text"`` (pre-analyze with - vision_analyze and prepend the description); see agent/image_routing.py. Gateway sessions - can carry /model overrides and image preprocessing runs before AIAgent sets the - auxiliary_client runtime globals, so resolve the per-session runtime bundle the upcoming - turn will use, not just the persisted default. + See agent/image_routing.py. Gateway sessions can carry /model overrides and image + preprocessing runs before AIAgent sets the auxiliary_client runtime globals, so resolve the + per-session runtime bundle the upcoming turn will use, not just the persisted default. """ try: from agent.image_routing import decide_image_input_mode @@ -2296,22 +2051,16 @@ class GatewayInboundMixin: resolved_requested_provider = "" needs_session_runtime = not resolved_provider or not resolved_model - has_session_identity = source is not None or session_key - if needs_session_runtime and has_session_identity: + if needs_session_runtime and (source is not None or session_key): try: turn_model, runtime_kwargs = self._resolve_session_agent_runtime( - source=source, - session_key=session_key, - user_config=cfg, + source=source, session_key=session_key, user_config=cfg, ) if not resolved_model and isinstance(turn_model, str): resolved_model = turn_model.strip() - runtime_provider = runtime_kwargs.get("provider") if isinstance(runtime_kwargs, dict) else None - runtime_requested_provider = ( - runtime_kwargs.get("requested_provider") - if isinstance(runtime_kwargs, dict) - else None - ) + rk = runtime_kwargs if isinstance(runtime_kwargs, dict) else {} + runtime_provider = rk.get("provider") + runtime_requested_provider = rk.get("requested_provider") if not resolved_provider and isinstance(runtime_provider, str): resolved_provider = runtime_provider.strip() if isinstance(runtime_requested_provider, str): @@ -2328,25 +2077,17 @@ class GatewayInboundMixin: resolved_model = _read_main_model() return decide_image_input_mode( - resolved_provider, - resolved_model, - cfg, - requested_provider=resolved_requested_provider, + resolved_provider, resolved_model, cfg, requested_provider=resolved_requested_provider, ) except Exception as exc: logger.debug("image_routing: decision failed, falling back to text — %s", exc) return "text" - async def _enrich_message_with_vision( - self, - user_text: str, - image_paths: List[str], - ) -> str: - """Auto-analyze user-attached images with the vision tool and prepend the descriptions to - the message text. + async def _enrich_message_with_vision(self, user_text: str, image_paths: List[str]) -> str: + """Auto-analyze user-attached images with the vision tool and prepend the descriptions. Description *and* local cache path are injected so the model understands the image without - a tool call and can re-examine it with vision_analyze. Returns the enriched message string. + a tool call and can re-examine it with vision_analyze. """ from tools.vision_tools import vision_analyze_tool from agent.memory_manager import sanitize_context @@ -2363,14 +2104,10 @@ class GatewayInboundMixin: for path in image_paths: try: logger.debug("Auto-analyzing user image: %s", path) - result_json = await vision_analyze_tool( - image_url=path, - user_prompt=analysis_prompt, - ) + result_json = await vision_analyze_tool(image_url=path, user_prompt=analysis_prompt) result = json.loads(result_json) if result.get("success"): - description = result.get("analysis", "") - description = sanitize_context(description) + description = sanitize_context(result.get("analysis", "")) enriched_parts.append( f"[The user sent an image~ Here's what I can see:\n{description}]\n" f"[If you need a closer look, use vision_analyze with " @@ -2390,26 +2127,66 @@ class GatewayInboundMixin: f"with vision_analyze using image_url: {path}]" ) - # Combine: vision descriptions first, then the user's original text - if enriched_parts: - prefix = "\n\n".join(enriched_parts) - if user_text: - return f"{prefix}\n\n{user_text}" - return prefix - return user_text + # Vision descriptions first, then the user's original text + if not enriched_parts: + return user_text + prefix = "\n\n".join(enriched_parts) + return f"{prefix}\n\n{user_text}" if user_text else prefix + + _EMPTY_TEXT_PLACEHOLDER = "(The user sent a message with no text content)" + + @classmethod + def _prepend_media_prefix(cls, prefix: str, user_text: str) -> str: + """``prefix`` + the user's text; the Discord empty-content placeholder is dropped as redundant.""" + if user_text and user_text.strip() != cls._EMPTY_TEXT_PLACEHOLDER: + return f"{prefix}\n\n{user_text}" + return prefix + + @staticmethod + def _untranscribed_audio_note(path: str) -> str: + """One minimal neutral marker for every STT failure. Never mention "no STT provider" or setup + steps — persisted in history they make the model keep volunteering STT-setup advice.""" + from tools.credential_files import to_agent_visible_cache_path + + agent_path = to_agent_visible_cache_path(os.path.abspath(path)) + return ( + "[voice message could not be transcribed automatically; " + f"the audio is available at: {agent_path}]" + ) + + async def _transcribe_one_clip(self, path: str, transcribe_audio, transcribe_audio_local_fallback) -> Tuple[Optional[str], str]: + """``(transcript_or_None, note)`` for one clip via configured STT with local fallback.""" + result = await asyncio.to_thread(transcribe_audio, path, None, "gateway") + if not result.get("success"): + fallback = await asyncio.to_thread(transcribe_audio_local_fallback, path) + if fallback.get("success"): + logger.info("Configured STT failed for %s; recovered with local STT", path) + result = fallback + if not result["success"]: + logger.info("Voice transcription failed for %s: %s", path, result.get("error", "unknown error")) + return None, self._untranscribed_audio_note(path) + transcript = result["transcript"] + # STT may return success=True with an empty/whitespace transcript (silence, cut-off); + # empty quotes make the agent reply to nothing and can loop, so emit a sentinel note. + if not (transcript or "").strip(): + return None, ( + "[The user sent a voice message but it came through " + "empty or inaudible — speech-to-text returned no " + "words. Do not guess at the content; ask the user " + "to resend or type it out.]" + ) + # Plain quoted line: a "The user sent a voice message..." wrapper read as a meta-instruction + # and made the LLM comment on voice mode instead. + return transcript, f'"{transcript}"' async def _enrich_message_with_transcription( - self, - user_text: str, - audio_paths: List[str], + self, user_text: str, audio_paths: List[str] ) -> tuple[str, List[str]]: - """Auto-transcribe user voice/audio messages using the configured STT provider and prepend - the transcript to the message text. + """Transcribe voice clips with the configured STT provider and prepend the transcripts. - Returns ``(enriched_text, successful_transcripts)``: the message with transcription - wrappers prepended, and the raw transcripts of successfully transcribed clips in input - order (empty if every clip failed or STT is disabled) so callers can echo them back to - the user before the agent loop. + Returns ``(enriched_text, successful_transcripts)``: transcripts of successfully + transcribed clips in input order (empty if every clip failed or STT is disabled) so callers + can echo them back before the agent loop. """ from gateway.run import _probe_audio_duration seen = set() @@ -2419,105 +2196,37 @@ class GatewayInboundMixin: for path in audio_paths: abs_path = os.path.abspath(path) duration_str = await _probe_audio_duration(abs_path) - if duration_str: - notes.append( - f"[The user sent a voice message: {abs_path} (duration: {duration_str})]" - ) - else: - notes.append(f"[The user sent a voice message: {abs_path}]") + suffix = f" (duration: {duration_str})" if duration_str else "" + notes.append(f"[The user sent a voice message: {abs_path}{suffix}]") if not notes: return user_text, [] - prefix = "\n\n".join(notes) - _placeholder = "(The user sent a message with no text content)" - if user_text and user_text.strip() == _placeholder: - return prefix, [] - if user_text: - return f"{prefix}\n\n{user_text}", [] - return prefix, [] + return self._prepend_media_prefix("\n\n".join(notes), user_text), [] try: from tools.transcription_tools import ( - transcribe_audio, - transcribe_audio_local_fallback, + transcribe_audio, transcribe_audio_local_fallback ) except ModuleNotFoundError as e: logger.error("Transcription module unavailable: %s", e) - unavailable_note = "[voice message could not be transcribed]" - _placeholder = "(The user sent a message with no text content)" - if user_text and user_text.strip() == _placeholder: - return unavailable_note, [] - if user_text: - return f"{unavailable_note}\n\n{user_text}", [] - return unavailable_note, [] + return self._prepend_media_prefix("[voice message could not be transcribed]", user_text), [] enriched_parts = [] successful_transcripts: List[str] = [] for path in audio_paths: try: logger.debug("Transcribing user voice: %s", path) - result = await asyncio.to_thread( - transcribe_audio, path, None, "gateway", + transcript, note = await self._transcribe_one_clip( + path, transcribe_audio, transcribe_audio_local_fallback, ) - if not result.get("success"): - fallback = await asyncio.to_thread( - transcribe_audio_local_fallback, - path, - ) - if fallback.get("success"): - logger.info( - "Configured STT failed for %s; recovered with local STT", - path, - ) - result = fallback - if result["success"]: - transcript = result["transcript"] - # STT may return success=True with an empty/whitespace transcript (silence, cut-off); - # empty quotes make the agent reply to nothing and can loop, so emit a sentinel note. - if not (transcript or "").strip(): - enriched_parts.append( - "[The user sent a voice message but it came through " - "empty or inaudible — speech-to-text returned no " - "words. Do not guess at the content; ask the user " - "to resend or type it out.]" - ) - continue + if transcript is not None: successful_transcripts.append(transcript) - # Pass the transcript as a plain quoted line: a "The user sent a voice message..." - # wrapper read as a meta-instruction and made the LLM comment on voice mode instead. - enriched_parts.append(f'"{transcript}"') - else: - error = result.get("error", "unknown error") - # All failure branches: one minimal neutral marker. Never mention "no STT provider", - # setup steps, or a DM sent — persisted in history they poison later turns (the model - # keeps volunteering STT-setup advice). Cause is logged for operators, not the prompt. - logger.info("Voice transcription failed for %s: %s", path, error) - from tools.credential_files import to_agent_visible_cache_path - - agent_path = to_agent_visible_cache_path(os.path.abspath(path)) - enriched_parts.append( - "[voice message could not be transcribed automatically; " - f"the audio is available at: {agent_path}]" - ) + enriched_parts.append(note) except Exception as e: logger.error("Transcription error: %s", e) - from tools.credential_files import to_agent_visible_cache_path - - agent_path = to_agent_visible_cache_path(os.path.abspath(path)) - enriched_parts.append( - "[voice message could not be transcribed automatically; " - f"the audio is available at: {agent_path}]" - ) + enriched_parts.append(self._untranscribed_audio_note(path)) if enriched_parts: - prefix = "\n\n".join(enriched_parts) - # Strip the empty-content placeholder from the Discord adapter - # when we successfully transcribed the audio — it's redundant. - _placeholder = "(The user sent a message with no text content)" - if user_text and user_text.strip() == _placeholder: - return prefix, successful_transcripts - if user_text: - return f"{prefix}\n\n{user_text}", successful_transcripts - return prefix, successful_transcripts + return self._prepend_media_prefix("\n\n".join(enriched_parts), user_text), successful_transcripts return user_text, successful_transcripts def _pending_event_audio_paths(self, event) -> List[str]: @@ -2531,9 +2240,7 @@ class GatewayInboundMixin: return audio_paths async def _transcribe_pending_audio_event_once( - self, - event, - user_text: Optional[str] = None, + self, event, user_text: Optional[str] = None ) -> tuple[str | None, List[str]]: """Transcribe a pending audio event once and cache the result on the event. @@ -2551,21 +2258,14 @@ class GatewayInboundMixin: text = user_text if user_text is not None else (getattr(event, "text", "") or "") enriched_text, successful_transcripts = await self._enrich_message_with_transcription( - text, - audio_paths, + text, audio_paths ) setattr(event, "_gateway_pending_stt_text", enriched_text) setattr(event, "_gateway_pending_stt_transcripts", list(successful_transcripts)) return enriched_text, successful_transcripts async def _echo_pending_stt_transcripts_once( - self, - event, - adapter, - source, - transcripts: List[str], - *, - metadata=None, + self, event, adapter, source, transcripts: List[str], *, metadata=None, log_context: str = "Transcript", ) -> None: """Echo pending-event STT transcripts to the chat at most once. @@ -2574,34 +2274,15 @@ class GatewayInboundMixin: ``merge_pending_message_event`` can append a second voice note and invalidate the cache, and the re-run returns earlier transcripts as a prefix, so only the unsent tail is echoed. """ - if ( - not transcripts - or not self._should_echo_stt_transcripts() - or adapter is None - ): + if not transcripts or not self._should_echo_stt_transcripts() or adapter is None: return already_echoed = int(getattr(event, "_gateway_pending_stt_echoed", 0) or 0) unsent = transcripts[already_echoed:] setattr(event, "_gateway_pending_stt_echoed", already_echoed + len(unsent)) - for tx in unsent: - try: - await adapter.send( - source.chat_id, - f'🎙️ "{tx}"', - metadata=metadata, - ) - except Exception as echo_exc: - logger.debug("%s echo failed (non-fatal): %s", log_context, echo_exc) + await self._echo_stt_transcripts(adapter, source, unsent, metadata=metadata, log_context=log_context) async def _transcribe_and_echo_pending_voice( - self, - event, - adapter, - source, - text: str, - *, - log_context: str, - metadata=_UNSET, + self, event, adapter, source, text: str, *, log_context: str, metadata=_UNSET ) -> tuple[str, List[str]]: """Transcribe a pending voice event and echo transcripts once. @@ -2613,20 +2294,13 @@ class GatewayInboundMixin: return text, [] try: enriched_text, transcripts = await self._transcribe_pending_audio_event_once( - event, - text, + event, text ) echo_meta = self._thread_metadata_for_source( - source, - self._reply_anchor_for_event(event), + source, self._reply_anchor_for_event(event) ) if metadata is _UNSET else metadata await self._echo_pending_stt_transcripts_once( - event, - adapter, - source, - transcripts, - metadata=echo_meta, - log_context=log_context, + event, adapter, source, transcripts, metadata=echo_meta, log_context=log_context ) return enriched_text or text, transcripts except Exception as trans_exc: diff --git a/gateway/run_notifications.py b/gateway/run_notifications.py index a20a93a08c..ff81650461 100644 --- a/gateway/run_notifications.py +++ b/gateway/run_notifications.py @@ -1,4 +1,4 @@ -"""Process/completion/update notifications, media delivery and async-delegation delivery methods for GatewayRunner. +"""Process/completion/update notifications, media delivery and async-delegation delivery for GatewayRunner. Split out of ``gateway/run.py``; bound onto ``GatewayRunner`` via the MRO. ``gateway.run`` internals are imported lazily inside method bodies (import cycle), @@ -7,17 +7,18 @@ so ``patch("gateway.run.X")`` keeps intercepting them at call time. from __future__ import annotations -import logging -from typing import TYPE_CHECKING import asyncio +import dataclasses import json +import logging import time from contextlib import suppress +from pathlib import Path +from typing import TYPE_CHECKING, Any, Dict, Optional, cast + from gateway.config import Platform, _BUILTIN_PLATFORM_VALUES from gateway.platforms.base import MessageEvent, MessageType from gateway.session import SessionEntry, SessionSource -from pathlib import Path -from typing import Any, Dict, Optional, cast if TYPE_CHECKING: # string annotations only; never imported at runtime (cycle) from gateway.run import GatewayRunner, TurnRunner # noqa: F401 @@ -25,9 +26,66 @@ if TYPE_CHECKING: # string annotations only; never imported at runtime (cycle) # Log-record parity with the origin module. logger = logging.getLogger("gateway.run") +_VIDEO_EXTS = {'.mp4', '.mov', '.avi', '.mkv', '.webm', '.3gp'} +# Routing fields copied verbatim from a process watcher onto its synthetic completion event. +_WATCHER_ROUTE_FIELDS = ("session_key", "platform", "chat_type", "chat_id", "thread_id", "user_id", "user_name") +_IMAGE_EXTS = {'.jpg', '.jpeg', '.png', '.webp', '.gif'} + +# Durable async-delegation claim transitions: kind -> (tools.async_delegation function, failure log). +_DURABLE_CLAIM_OPS = { + "drop": ("drop_completion_delivery", "Could not drop durable completion claim"), + "release": ("release_completion_delivery", "Could not release durable completion claim"), +} + class GatewayNotificationsMixin: - """Process/completion/update notifications, media delivery and async-delegation delivery methods for GatewayRunner.""" + """Process/completion/update notifications, media delivery and async-delegation delivery for GatewayRunner.""" + + # Routing-complete coalescing keys: process completions (short-window fan-in) and async + # delegations (originating session + parent session + route). + _COMPLETION_BATCH_KEY_FIELDS = ("session_key", "platform", "chat_type", "chat_id", "thread_id", "user_id") + _ASYNC_GROUP_KEY_FIELDS = ("session_key", "parent_session_id", *_COMPLETION_BATCH_KEY_FIELDS[1:]) + + @dataclasses.dataclass + class _UpdatePaths: + """Marker files ``hermes update --gateway`` and its watcher exchange under HERMES_HOME.""" + + pending: Path + claimed: Path + output: Path + exit_code: Path + prompt: Path + response: Path + + def any_pending(self) -> bool: + return self.pending.exists() or self.claimed.exists() + + def unlink_all(self) -> None: + for p in (self.pending, self.claimed, self.output, self.exit_code, self.prompt, self.response): + p.unlink(missing_ok=True) + + @dataclasses.dataclass + class _UpdateTarget: + """Resolved delivery target for update watcher messages.""" + + adapter: Any + chat_id: Any + session_key: Optional[str] + metadata: Any + platform: Any + + def send_metadata(self): + from gateway.run import _non_conversational_metadata + return _non_conversational_metadata(self.metadata, platform=self.platform) + + @dataclasses.dataclass + class _CompletionClaim: + """Pre-flight outcome for one completion delivery.""" + + delegation_id: str = "" + claim_id: str = "" + proceed: bool = True + early_result: Optional[bool] = None async def _deliver_platform_notice(self, source, content: str) -> None: """Deliver a setup/operational notice using platform-specific privacy rules.""" @@ -37,15 +95,9 @@ class GatewayNotificationsMixin: return config = getattr(self, "config", None) - if ( - config - and getattr(source, "platform", None) == Platform.SLACK - and _is_slack_ignored_channel(config, getattr(source, "chat_id", None)) - ): - logger.info( - "Skipping Slack platform notice for configured ignored channel %s", - getattr(source, "chat_id", None), - ) + chat_id = getattr(source, "chat_id", None) + if config and getattr(source, "platform", None) == Platform.SLACK and _is_slack_ignored_channel(config, chat_id): + logger.info("Skipping Slack platform notice for configured ignored channel %s", chat_id) return notice_delivery = "public" @@ -55,27 +107,73 @@ class GatewayNotificationsMixin: metadata = self._thread_metadata_for_source(source) if notice_delivery == "private" and getattr(source, "user_id", None): try: - result = await adapter.send_private_notice( - source.chat_id, - source.user_id, - content, - metadata=metadata, - ) + result = await adapter.send_private_notice(source.chat_id, source.user_id, content, metadata=metadata) if getattr(result, "success", False): return except Exception: logger.debug( "[%s] send_private_notice failed, falling back to public", - getattr(source, "platform", "?"), - exc_info=True, + getattr(source, "platform", "?"), exc_info=True, ) await adapter.send(source.chat_id, content, metadata=metadata) + async def _resolve_compression_lineage_target( + self, session_db: Any, session_entry: SessionEntry, pinned_session_id: str, + ) -> Optional[str]: + """Return the live compression tip of ``pinned_session_id`` if the route owns that lineage, else None.""" + try: + target_session_id = await session_db.get_compression_tip(pinned_session_id) + except Exception: + logger.debug("Async-delegation compression-tip lookup failed for %s", pinned_session_id, exc_info=True) + target_session_id = None + + if not target_session_id or target_session_id == pinned_session_id: + logger.warning( + "Async-delegation completion pinned to compressed session %s " + "without a continuation; dropping injection.", pinned_session_id, + ) + return None + + try: + tip_row = await session_db.get_session(target_session_id) + except Exception: + tip_row = None + if tip_row is None or tip_row.get("ended_at"): + logger.warning( + "Async-delegation compression continuation %s is %s; dropping injection.", + target_session_id, "unknown" if tip_row is None else "ended", + ) + return None + + route_owns_lineage = session_entry.session_id in {pinned_session_id, target_session_id} + if not route_owns_lineage: + # A delegation may survive several compression rotations: accept an intermediate stale + # route only when its own verified compression tip is the same live target. + try: + route_row = await session_db.get_session(session_entry.session_id) + route_tip = ( + await session_db.get_compression_tip(session_entry.session_id) + if route_row is not None + and route_row.get("ended_at") + and route_row.get("end_reason") == "compression" + else None + ) + except Exception: + route_tip = None + route_owns_lineage = route_tip == target_session_id + + if not route_owns_lineage: + logger.warning( + "Async-delegation completion for compression lineage %s -> %s " + "does not own current route %s; dropping injection.", + pinned_session_id, target_session_id, session_entry.session_id, + ) + return None + return target_session_id + async def _resolve_async_delegation_session( - self, - session_entry: SessionEntry, - pinned_session_id: str, + self, session_entry: SessionEntry, pinned_session_id: str, ) -> Optional[SessionEntry]: """Resolve an async completion to its verified owning gateway session. @@ -96,17 +194,12 @@ class GatewayNotificationsMixin: try: pinned_row = await session_db.get_session(pinned_session_id) except Exception: - logger.debug( - "Async-delegation parent lookup failed for %s", - pinned_session_id, - exc_info=True, - ) + logger.debug("Async-delegation parent lookup failed for %s", pinned_session_id, exc_info=True) if pinned_row is None: logger.warning( "Async-delegation completion has unknown spawning session %s; " - "dropping injection (#55578 fail-closed).", - pinned_session_id, + "dropping injection (#55578 fail-closed).", pinned_session_id, ) return None @@ -118,9 +211,7 @@ class GatewayNotificationsMixin: logger.warning( "Async-delegation completion pinned to user-closed session %s " "(end_reason=%r); dropping injection instead of resurrecting it " - "(#55578 fail-closed).", - pinned_session_id, - _end_reason, + "(#55578 fail-closed).", pinned_session_id, _end_reason, ) return None if _end_reason != "compression": @@ -130,75 +221,15 @@ class GatewayNotificationsMixin: logger.info( "Async-delegation completion pinned to %s-ended session %s; " "retargeting to the chat's current session %s.", - _end_reason or "idle", - pinned_session_id, - session_entry.session_id, + _end_reason or "idle", pinned_session_id, session_entry.session_id, ) return session_entry follows_compression = True - try: - target_session_id = await session_db.get_compression_tip( - pinned_session_id - ) - except Exception: - logger.debug( - "Async-delegation compression-tip lookup failed for %s", - pinned_session_id, - exc_info=True, - ) - target_session_id = None - - if not target_session_id or target_session_id == pinned_session_id: - logger.warning( - "Async-delegation completion pinned to compressed session %s " - "without a continuation; dropping injection.", - pinned_session_id, - ) - return None - - try: - tip_row = await session_db.get_session(target_session_id) - except Exception: - tip_row = None - if tip_row is None or tip_row.get("ended_at"): - logger.warning( - "Async-delegation compression continuation %s is %s; " - "dropping injection.", - target_session_id, - "unknown" if tip_row is None else "ended", - ) - return None - - route_owns_lineage = session_entry.session_id in { - pinned_session_id, - target_session_id, - } - if not route_owns_lineage: - # A long-running delegation may survive multiple compression - # rotations. Accept an intermediate stale route only when its - # own verified compression tip is the same live target. - try: - route_row = await session_db.get_session(session_entry.session_id) - route_tip = ( - await session_db.get_compression_tip(session_entry.session_id) - if route_row is not None - and route_row.get("ended_at") - and route_row.get("end_reason") == "compression" - else None - ) - except Exception: - route_tip = None - route_owns_lineage = route_tip == target_session_id - - if not route_owns_lineage: - logger.warning( - "Async-delegation completion for compression lineage %s -> %s " - "does not own current route %s; dropping injection.", - pinned_session_id, - target_session_id, - session_entry.session_id, - ) + target_session_id = await self._resolve_compression_lineage_target( + session_db, session_entry, pinned_session_id, + ) + if target_session_id is None: return None if target_session_id == session_entry.session_id: @@ -207,102 +238,69 @@ class GatewayNotificationsMixin: prior_session_id = session_entry.session_id if follows_compression: switched = await self.async_session_store.advance_compression_session( - session_entry.session_key, - prior_session_id, - target_session_id, + session_entry.session_key, prior_session_id, target_session_id, ) else: - switched = await self.async_session_store.switch_session( - session_entry.session_key, - target_session_id, - ) + switched = await self.async_session_store.switch_session(session_entry.session_key, target_session_id) if switched is None: logger.warning( "Async-delegation completion could not bind routing key %s to " "owning session %s; dropping injection.", - session_entry.session_key, - target_session_id, + session_entry.session_key, target_session_id, ) return None logger.info( "Pinned async-delegation completion to owning session %s " "(was %s) for routing key %s (#57498)", - target_session_id, - prior_session_id, - session_entry.session_key, + target_session_id, prior_session_id, session_entry.session_key, ) return switched async def _deliver_media_from_response( - self, - response: str, - event: MessageEvent, - adapter, + self, response: str, event: MessageEvent, adapter, thread_metadata: Optional[Dict[str, Any]] = None, ) -> None: """Extract explicit MEDIA: tags from an already-streamed response and deliver them. - The text is already delivered; this only handles file attachments the normal - _process_message_background path would have caught. Unlike the non-streaming path in - ``gateway/platforms/base.py`` this rescan is EXPLICIT-ONLY: a bare local path in a - streamed reply was either shown as text or is stale inspected content, and promoting it - sent files the model never asked to deliver. + The text is already delivered; this only handles file attachments. Unlike the non-streaming + path in ``gateway/platforms/base.py`` this rescan is EXPLICIT-ONLY: a bare local path in a + streamed reply was either shown as text or is stale inspected content, and promoting it sent + files the model never asked to deliver. MEDIA tags are NOT deduped against prior turns: a + MEDIA: directive in the final reply is a deliberate attach (incl. user-requested resends); + stale auto-appended tags are deduped upstream (_collect_auto_append_media_tags). """ - from pathlib import Path from urllib.parse import quote as _quote - try: - # Capture [[as_document]] before extract_media strips it, so the dispatch partition - # below can route image-extension files through send_document (preserving bytes) instead - # of send_multiple_images (Telegram sendPhoto recompresses to ~1280px). + # Capture [[as_document]] before extract_media strips it: image files then go through + # send_document (preserving bytes), not send_multiple_images (Telegram recompresses). force_document_attachments = "[[as_document]]" in response from gateway.platforms.base import BasePlatformAdapter, should_send_media_as_audio - media_files, cleaned = adapter.extract_media(response) media_files = BasePlatformAdapter.filter_media_delivery_paths(media_files) - # Do NOT deduplicate explicit MEDIA tags against prior turns here. This rescan is - # already EXPLICIT-ONLY (see docstring): a MEDIA: directive in the final streamed reply - # is the model deliberately attaching a file — including a user-requested resend. Stale - # auto-appended tags are deduped upstream (_collect_auto_append_media_tags). Strip image - # URLs for parity with the non-streaming chain, but do NOT run extract_local_files here. + # Strip image URLs for parity with the non-streaming chain; no extract_local_files here. adapter.extract_images(cleaned) _thread_meta = ( dict(thread_metadata) if thread_metadata is not None - else self._thread_metadata_for_source( - event.source, - self._reply_anchor_for_event(event), - ) + else self._thread_metadata_for_source(event.source, self._reply_anchor_for_event(event)) ) + chat_id = event.source.chat_id - _VIDEO_EXTS = {'.mp4', '.mov', '.avi', '.mkv', '.webm', '.3gp'} - _IMAGE_EXTS = {'.jpg', '.jpeg', '.png', '.webp', '.gif'} - - # Partition out images so they can be sent as a single batch (e.g. Signal's multi- - # attachment RPC). When [[as_document]] was set, image-extension files skip the photo - # path and route to send_document below — preserving original bytes. - image_paths: list = [] - non_image_media: list = [] - for media_path, is_voice in media_files: + # Images go out as one batch (e.g. Signal's multi-attachment RPC) unless [[as_document]]. + def _is_photo(media_path: str, is_voice: bool) -> bool: ext = Path(media_path).suffix.lower() - if (ext in _IMAGE_EXTS - and not is_voice - and not force_document_attachments): - image_paths.append(media_path) - else: - non_image_media.append((media_path, is_voice)) + return ext in _IMAGE_EXTS and not is_voice and not force_document_attachments + + image_paths = [p for p, v in media_files if _is_photo(p, v)] + non_image_media = [(p, v) for p, v in media_files if not _is_photo(p, v)] if image_paths: try: images = [(f"file://{_quote(p)}", "") for p in image_paths] - await adapter.send_multiple_images( - chat_id=event.source.chat_id, - images=images, - metadata=_thread_meta, - ) + await adapter.send_multiple_images(chat_id=chat_id, images=images, metadata=_thread_meta) except Exception as e: logger.warning("[%s] Post-stream image batch delivery failed: %s", adapter.name, e) @@ -311,23 +309,12 @@ class GatewayNotificationsMixin: ext = Path(media_path).suffix.lower() if should_send_media_as_audio(event.source.platform, ext, is_voice=is_voice): await adapter.send_voice( - chat_id=event.source.chat_id, - audio_path=media_path, - metadata=_thread_meta, - is_voice=is_voice, + chat_id=chat_id, audio_path=media_path, metadata=_thread_meta, is_voice=is_voice, ) elif ext in _VIDEO_EXTS: - await adapter.send_video( - chat_id=event.source.chat_id, - video_path=media_path, - metadata=_thread_meta, - ) + await adapter.send_video(chat_id=chat_id, video_path=media_path, metadata=_thread_meta) else: - await adapter.send_document( - chat_id=event.source.chat_id, - file_path=media_path, - metadata=_thread_meta, - ) + await adapter.send_document(chat_id=chat_id, file_path=media_path, metadata=_thread_meta) except Exception as e: logger.warning("[%s] Post-stream media delivery failed: %s", adapter.name, e) @@ -335,25 +322,18 @@ class GatewayNotificationsMixin: logger.warning("Post-stream media extraction failed: %s", e) async def _deliver_queued_first_response( - self, - response: str, - source: SessionSource, - adapter, - metadata: Optional[Dict[str, Any]] = None, - event_message_id: Optional[str] = None, - text_already_delivered: bool = False, - deliver_media: bool = True, - stream_consumer=None, + self, response: str, source: SessionSource, adapter, + metadata: Optional[Dict[str, Any]] = None, event_message_id: Optional[str] = None, + text_already_delivered: bool = False, deliver_media: bool = True, stream_consumer=None, ) -> None: """Deliver a queued response using the normal text+attachment split.""" from gateway.run import _strip_response_attachments_for_direct_send if not text_already_delivered: text_content = _strip_response_attachments_for_direct_send(response, adapter) if text_content: - # Reconcile-by-edit first (live finding, 2026-08-16 canary): when the stream - # consumer delivered/sealed a message but its recorded payload didn't confirm the - # final (post-stream mutation), plain-sending here creates the duplicate — the - # sealed message already carries most of the answer. + # Reconcile-by-edit first: a stream-sealed message whose recorded payload didn't confirm + # the final (post-stream mutation) already carries most of the answer; a plain send here + # would duplicate it. _reconciled = False _sc_msg_id = getattr(stream_consumer, "message_id", None) if ( @@ -363,10 +343,7 @@ class GatewayNotificationsMixin: ): try: _edit_res = await adapter.edit_message( - chat_id=source.chat_id, - message_id=_sc_msg_id, - content=text_content, - finalize=True, + chat_id=source.chat_id, message_id=_sc_msg_id, content=text_content, finalize=True, ) if getattr(_edit_res, "success", False): _reconciled = True @@ -375,34 +352,17 @@ class GatewayNotificationsMixin: _sc_msg_id, ) except Exception as _qe: - logger.debug( - "Queued-lane reconcile edit failed (%s); falling back to send.", - _qe, - ) + logger.debug("Queued-lane reconcile edit failed (%s); falling back to send.", _qe) if not _reconciled: - await adapter.send( - source.chat_id, - text_content, - metadata=metadata, - ) + await adapter.send(source.chat_id, text_content, metadata=metadata) - # Failed turns still deliver their (normalized failure) text above, but must not upload - # attachments as if the turn succeeded — mirrors the ``not agent_result.get("failed")`` - # guard on the completed-turn delivery path. + # Failed turns deliver their (normalized failure) text but must not upload attachments as if + # they succeeded — mirrors the ``not agent_result.get("failed")`` completed-turn guard. if not deliver_media: return - synthetic_event = MessageEvent( - text="", - source=source, - message_id=event_message_id, - ) - await self._deliver_media_from_response( - response, - synthetic_event, - adapter, - thread_metadata=metadata, - ) + synthetic_event = MessageEvent(text="", source=source, message_id=event_message_id) + await self._deliver_media_from_response(response, synthetic_event, adapter, thread_metadata=metadata) def _schedule_update_notification_watch(self) -> None: """Ensure a background task is watching for update completion.""" @@ -411,362 +371,284 @@ class GatewayNotificationsMixin: return try: - self._update_notification_task = asyncio.create_task( - self._watch_update_progress() - ) + self._update_notification_task = asyncio.create_task(self._watch_update_progress()) except RuntimeError: logger.debug("Skipping update notification watcher: no running event loop") + @classmethod + def _update_paths(cls) -> "GatewayNotificationsMixin._UpdatePaths": + from gateway.run import _hermes_home + return cls._UpdatePaths( + pending=_hermes_home / ".update_pending.json", + claimed=_hermes_home / ".update_pending.claimed.json", + output=_hermes_home / ".update_output.txt", + exit_code=_hermes_home / ".update_exit_code", + prompt=_hermes_home / ".update_prompt.json", response=_hermes_home / ".update_response", + ) + + def _resolve_update_target(self, paths: "_UpdatePaths") -> Optional["_UpdateTarget"]: + """Resolve adapter/chat/session for update watcher messages from the pending marker.""" + for path in (paths.claimed, paths.pending): + if not path.exists(): + continue + try: + pending = json.loads(path.read_text(encoding="utf-8")) + platform_str = pending.get("platform") + chat_id = pending.get("chat_id") + session_key = pending.get("session_key") + if not (platform_str and chat_id): + return None + platform = Platform(platform_str) + adapter = self.adapters.get(platform) + metadata = self._thread_metadata_for_target( + platform, chat_id, pending.get("thread_id"), chat_type=pending.get("chat_type"), + reply_to_message_id=pending.get("message_id"), adapter=adapter, + ) + if not adapter: + return None + # Fallback session key if not stored (old pending files) + return self._UpdateTarget( + adapter, chat_id, session_key or f"{platform_str}:{chat_id}", metadata, platform, + ) + except Exception: + pass + return None + + async def _watch_update_completion_only(self, paths: "_UpdatePaths", deadline: float, poll_interval: float) -> None: + """Fallback when no adapter/chat can be resolved: wait for the exit code, then notify.""" + logger.warning("Update watcher: cannot resolve adapter/chat_id, falling back to completion-only") + # Keep polling until _send_update_notification actually delivers (True) — it re-resolves the + # adapter each call and returns False (markers kept) while the platform is still reconnecting. + loop = asyncio.get_running_loop() + while paths.any_pending() and loop.time() < deadline: + if paths.exit_code.exists() and await self._send_update_notification(): + return + await asyncio.sleep(poll_interval) + if paths.any_pending() and not paths.exit_code.exists(): + paths.exit_code.write_text("124", encoding="utf-8") + await self._send_update_notification() + + @staticmethod + def _read_update_output_since(path: Path, offset: int) -> tuple[str, int]: + """Read update output defensively; logs may contain invalid UTF-8.""" + try: + data = path.read_bytes() + except OSError: + return "", offset + if len(data) <= offset: + return "", len(data) + return data[offset:].decode("utf-8", errors="replace"), len(data) + + async def _send_update_output(self, target: "_UpdateTarget", text: str) -> None: + """Send buffered update output as fenced chunks that fit message limits (Telegram: 4096).""" + from tools.ansi_strip import strip_ansi + clean = strip_ansi(text).strip() + if not clean: + return + max_chunk = 3500 + for i in range(0, len(clean), max_chunk): + try: + await target.adapter.send( + target.chat_id, f"```\n{clean[i:i + max_chunk]}\n```", metadata=target.send_metadata(), + ) + except Exception as e: + logger.debug("Update stream send failed: %s", e) + + async def _forward_update_prompt(self, target: "_UpdateTarget", prompt_text: str, default: str) -> None: + """Forward an update prompt: platform-native buttons first (Discord, Telegram), else text.""" + sent_buttons = False + adapter = target.adapter + if getattr(type(adapter), "send_update_prompt", None) is not None: + try: + await adapter.send_update_prompt( + chat_id=target.chat_id, prompt=prompt_text, default=default, + session_key=target.session_key, metadata=target.send_metadata(), + ) + sent_buttons = True + except Exception as btn_err: + logger.debug("Button-based update prompt failed: %s", btn_err) + if not sent_buttons: + default_hint = f" (default: {default})" if default else "" + _p = getattr(adapter, "typed_command_prefix", "/") + await adapter.send( + target.chat_id, + f"⚕ **Update needs your input:**\n\n{prompt_text}{default_hint}\n\n" + f"Reply `{_p}approve` (yes) or `{_p}deny` (no), or type your answer directly.", + metadata=target.send_metadata(), + ) + # Keep the prompt marker on disk until the user answers so a watcher after a mid-prompt + # gateway restart can recover by re-forwarding it. + self._session_state(target.session_key).persistent.update_prompt_pending = True + logger.info("Forwarded update prompt to %s: %s", target.session_key, prompt_text[:80]) + + def _clear_update_markers(self, paths: "_UpdatePaths", session_key: Optional[str]) -> None: + paths.unlink_all() + state = self._peek_session_state(session_key) + if state is not None: + state.persistent.update_prompt_pending = False + async def _watch_update_progress( - self, - poll_interval: float = 2.0, - stream_interval: float = 4.0, - timeout: float = 1800.0, + self, poll_interval: float = 2.0, stream_interval: float = 4.0, timeout: float = 1800.0 ) -> None: """Watch ``hermes update --gateway``, streaming output + forwarding prompts. Polls ``.update_output.txt`` for new content and sends chunks to the user periodically; detects ``.update_prompt.json`` (written when the update process needs input) and forwards it. """ - from gateway.run import _hermes_home, _non_conversational_metadata - pending_path = _hermes_home / ".update_pending.json" - claimed_path = _hermes_home / ".update_pending.claimed.json" - output_path = _hermes_home / ".update_output.txt" - exit_code_path = _hermes_home / ".update_exit_code" - prompt_path = _hermes_home / ".update_prompt.json" - + paths = self._update_paths() loop = asyncio.get_running_loop() deadline = loop.time() + timeout - # Resolve the adapter and chat_id for sending messages - adapter = None - chat_id = None - session_key = None - metadata = None - for path in (claimed_path, pending_path): - if path.exists(): - try: - pending = json.loads(path.read_text(encoding="utf-8")) - platform_str = pending.get("platform") - chat_id = pending.get("chat_id") - chat_type = pending.get("chat_type") - session_key = pending.get("session_key") - thread_id = pending.get("thread_id") - message_id = pending.get("message_id") - if platform_str and chat_id: - platform = Platform(platform_str) - adapter = self.adapters.get(platform) - metadata = self._thread_metadata_for_target( - platform, - chat_id, - thread_id, - chat_type=chat_type, - reply_to_message_id=message_id, - adapter=adapter, - ) - # Fallback session key if not stored (old pending files) - if not session_key: - session_key = f"{platform_str}:{chat_id}" - break - except Exception: - pass - - if not adapter or not chat_id: - logger.warning("Update watcher: cannot resolve adapter/chat_id, falling back to completion-only") - # Completion-only fallback: wait for the exit code, then keep polling until - # _send_update_notification actually delivers (True) — it re-resolves the adapter each - # call and returns False (markers kept) while the platform is still reconnecting. - while (pending_path.exists() or claimed_path.exists()) and loop.time() < deadline: - if exit_code_path.exists() and await self._send_update_notification(): - return - await asyncio.sleep(poll_interval) - if (pending_path.exists() or claimed_path.exists()) and not exit_code_path.exists(): - exit_code_path.write_text("124", encoding="utf-8") - await self._send_update_notification() + target = self._resolve_update_target(paths) + if target is None: + await self._watch_update_completion_only(paths, deadline, poll_interval) return - - def _strip_ansi(text: str) -> str: - from tools.ansi_strip import strip_ansi - return strip_ansi(text) - - def _read_output_since(path: Path, offset: int) -> tuple[str, int]: - """Read update output defensively; logs may contain invalid UTF-8.""" - try: - data = path.read_bytes() - except OSError: - return "", offset - if len(data) <= offset: - return "", len(data) - return data[offset:].decode("utf-8", errors="replace"), len(data) + session_key = target.session_key bytes_sent = 0 last_stream_time = loop.time() buffer = "" async def _flush_buffer() -> None: - """Send buffered output to the user.""" nonlocal buffer, last_stream_time - if not buffer.strip(): - buffer = "" - return - # Chunk to fit message limits (Telegram: 4096, others: generous) - clean = _strip_ansi(buffer).strip() - buffer = "" - last_stream_time = loop.time() - if not clean: - return - # Split into chunks if too long - max_chunk = 3500 - chunks = [clean[i:i + max_chunk] for i in range(0, len(clean), max_chunk)] - for chunk in chunks: + text, buffer = buffer, "" + if text.strip(): + last_stream_time = loop.time() + await self._send_update_output(target, text) + + def _read_new_output() -> None: + nonlocal buffer, bytes_sent + if paths.output.exists(): try: - await adapter.send( - chat_id, - f"```\n{chunk}\n```", - metadata=_non_conversational_metadata(metadata, platform=platform), - ) - except Exception as e: - logger.debug("Update stream send failed: %s", e) - - while loop.time() < deadline: - # Check for completion - if exit_code_path.exists(): - # Read any remaining output - if output_path.exists(): - try: - chunk, bytes_sent = _read_output_since(output_path, bytes_sent) - if chunk: - buffer += chunk - except OSError: - pass - await _flush_buffer() - - # Send final status - try: - exit_code_raw = exit_code_path.read_text(encoding="utf-8").strip() or "1" - exit_code = int(exit_code_raw) - if exit_code == 0: - await adapter.send( - chat_id, - "✅ Hermes update finished.", - metadata=_non_conversational_metadata(metadata, platform=platform), - ) - else: - await adapter.send( - chat_id, - "❌ Hermes update failed (exit code {}).".format(exit_code), - metadata=_non_conversational_metadata(metadata, platform=platform), - ) - logger.info("Update finished (exit=%s), notified %s", exit_code, session_key) - except Exception as e: - logger.warning("Update final notification failed: %s", e) - - # Cleanup - for p in (pending_path, claimed_path, output_path, - exit_code_path, prompt_path): - p.unlink(missing_ok=True) - (_hermes_home / ".update_response").unlink(missing_ok=True) - _up_done = self._peek_session_state(session_key) - if _up_done is not None: - _up_done.persistent.update_prompt_pending = False - return - - # Check for new output - if output_path.exists(): - try: - chunk, bytes_sent = _read_output_since(output_path, bytes_sent) - if chunk: - buffer += chunk + chunk, bytes_sent = self._read_update_output_since(paths.output, bytes_sent) + buffer += chunk except OSError: pass - # Flush buffer periodically + while loop.time() < deadline: + if paths.exit_code.exists(): + _read_new_output() + await _flush_buffer() + try: + exit_code = int(paths.exit_code.read_text(encoding="utf-8").strip() or "1") + await target.adapter.send( + target.chat_id, + "✅ Hermes update finished." if exit_code == 0 + else "❌ Hermes update failed (exit code {}).".format(exit_code), + metadata=target.send_metadata(), + ) + logger.info("Update finished (exit=%s), notified %s", exit_code, session_key) + except Exception as e: + logger.warning("Update final notification failed: %s", e) + self._clear_update_markers(paths, session_key) + return + + _read_new_output() if buffer.strip() and (loop.time() - last_stream_time) >= stream_interval: await _flush_buffer() - # Check for prompts — only forward if we haven't already sent one that's still awaiting - # a response. Without this guard the watcher would re-read the same .update_prompt.json - # every poll cycle and spam the user with duplicate prompt messages. - _up_pending_state = ( - self._peek_session_state(session_key) if session_key else None - ) - if (prompt_path.exists() and session_key - and not ( - _up_pending_state is not None - and _up_pending_state.persistent.update_prompt_pending - )): + # Forward a prompt only when none is still awaiting a response; otherwise the watcher + # would re-read the same .update_prompt.json every poll and spam duplicate prompts. + _pending_state = self._peek_session_state(session_key) if session_key else None + if ( + paths.prompt.exists() and session_key + and not (_pending_state is not None and _pending_state.persistent.update_prompt_pending) + ): try: - prompt_data = json.loads(prompt_path.read_text(encoding="utf-8")) + prompt_data = json.loads(paths.prompt.read_text(encoding="utf-8")) prompt_text = prompt_data.get("prompt", "") - default = prompt_data.get("default", "") if prompt_text: - # Flush any buffered output first so the user sees - # context before the prompt - await _flush_buffer() - # Try platform-native buttons first (Discord, Telegram) - sent_buttons = False - if getattr(type(adapter), "send_update_prompt", None) is not None: - try: - await adapter.send_update_prompt( - chat_id=chat_id, - prompt=prompt_text, - default=default, - session_key=session_key, - metadata=_non_conversational_metadata(metadata, platform=platform), - ) - sent_buttons = True - except Exception as btn_err: - logger.debug("Button-based update prompt failed: %s", btn_err) - if not sent_buttons: - default_hint = f" (default: {default})" if default else "" - _p = getattr(adapter, "typed_command_prefix", "/") - await adapter.send( - chat_id, - f"⚕ **Update needs your input:**\n\n" - f"{prompt_text}{default_hint}\n\n" - f"Reply `{_p}approve` (yes) or `{_p}deny` (no), " - f"or type your answer directly.", - metadata=_non_conversational_metadata(metadata, platform=platform), - ) - # Keep the prompt marker on disk until the user answers so a watcher after a - # mid-prompt gateway restart can recover by re-forwarding it. - self._session_state( - session_key - ).persistent.update_prompt_pending = True - # .update_response to continue — it doesn't re-check - logger.info("Forwarded update prompt to %s: %s", session_key, prompt_text[:80]) + await _flush_buffer() # user sees context before the prompt + await self._forward_update_prompt(target, prompt_text, prompt_data.get("default", "")) except (json.JSONDecodeError, OSError) as e: logger.debug("Failed to read update prompt: %s", e) await asyncio.sleep(poll_interval) - # Timeout - if not exit_code_path.exists(): + if not paths.exit_code.exists(): logger.warning("Update watcher timed out after %.0fs", timeout) - exit_code_path.write_text("124", encoding="utf-8") + paths.exit_code.write_text("124", encoding="utf-8") await _flush_buffer() with suppress(Exception): - await adapter.send( - chat_id, - "❌ Hermes update timed out after 30 minutes.", - metadata=_non_conversational_metadata(metadata, platform=platform), + await target.adapter.send( + target.chat_id, "❌ Hermes update timed out after 30 minutes.", metadata=target.send_metadata(), ) - for p in (pending_path, claimed_path, output_path, - exit_code_path, prompt_path): - p.unlink(missing_ok=True) - (_hermes_home / ".update_response").unlink(missing_ok=True) - _up_timeout_state = self._peek_session_state(session_key) - if _up_timeout_state is not None: - _up_timeout_state.persistent.update_prompt_pending = False + self._clear_update_markers(paths, session_key) async def _send_update_notification(self) -> bool: """If an update finished, notify the user. False while the update is still running (caller may retry); True after a definitive send/skip. """ - from gateway.run import _hermes_home, _non_conversational_metadata - pending_path = _hermes_home / ".update_pending.json" - claimed_path = _hermes_home / ".update_pending.claimed.json" - output_path = _hermes_home / ".update_output.txt" - exit_code_path = _hermes_home / ".update_exit_code" - - if not pending_path.exists() and not claimed_path.exists(): + from gateway.run import _non_conversational_metadata + paths = self._update_paths() + if not paths.any_pending(): return False cleanup = True - active_pending_path = claimed_path + active_pending_path = paths.claimed + + def _defer(reason: str, *args) -> bool: + nonlocal cleanup, active_pending_path + logger.info(reason, *args) + cleanup = False + active_pending_path = paths.pending + paths.claimed.replace(paths.pending) + return False + try: - if pending_path.exists(): + if paths.pending.exists(): try: - pending_path.replace(claimed_path) + paths.pending.replace(paths.claimed) except FileNotFoundError: - if not claimed_path.exists(): + if not paths.claimed.exists(): return True - elif not claimed_path.exists(): + elif not paths.claimed.exists(): return True - pending = json.loads(claimed_path.read_text(encoding="utf-8")) + pending = json.loads(paths.claimed.read_text(encoding="utf-8")) platform_str = pending.get("platform") chat_id = pending.get("chat_id") - chat_type = pending.get("chat_type") - thread_id = pending.get("thread_id") - message_id = pending.get("message_id") - if not exit_code_path.exists(): - logger.info("Update notification deferred: update still running") - cleanup = False - active_pending_path = pending_path - claimed_path.replace(pending_path) - return False + if not paths.exit_code.exists(): + return _defer("Update notification deferred: update still running") - exit_code_raw = exit_code_path.read_text(encoding="utf-8").strip() or "1" - exit_code = int(exit_code_raw) + exit_code = int(paths.exit_code.read_text(encoding="utf-8").strip() or "1") + output = paths.output.read_bytes().decode("utf-8", errors="replace") if paths.output.exists() else "" - # Read the captured update output - output = "" - if output_path.exists(): - output = output_path.read_bytes().decode("utf-8", errors="replace") - - # Resolve adapter platform = Platform(platform_str) adapter = self.adapters.get(platform) if not adapter and chat_id: # The update finished, but the target platform has not reconnected yet (common right - # after the restart that `hermes update` triggers). Treating "adapter missing" as a - # definitive skip would delete the markers and silently lose the notification; - # preserve them so a later retry (watcher poll or next startup) can deliver it. - logger.info( - "Update notification deferred: %s adapter not connected yet", - platform_str, - ) - cleanup = False - active_pending_path = pending_path - claimed_path.replace(pending_path) - return False + # after the restart that `hermes update` triggers). A definitive skip would delete the + # markers and silently lose the notification; preserve them for a later retry. + return _defer("Update notification deferred: %s adapter not connected yet", platform_str) if adapter and chat_id: metadata = self._thread_metadata_for_target( - platform, - chat_id, - thread_id, - chat_type=chat_type, - reply_to_message_id=message_id, - adapter=adapter, + platform, chat_id, pending.get("thread_id"), chat_type=pending.get("chat_type"), + reply_to_message_id=pending.get("message_id"), adapter=adapter, ) - # Strip ANSI escape codes for clean display from tools.ansi_strip import strip_ansi output = strip_ansi(output).strip() if output: if len(output) > 3500: output = "…" + output[-3500:] - if exit_code == 0: - msg = f"✅ Hermes update finished.\n\n```\n{output}\n```" - else: - msg = f"❌ Hermes update failed.\n\n```\n{output}\n```" + status = "✅ Hermes update finished." if exit_code == 0 else "❌ Hermes update failed." + msg = f"{status}\n\n```\n{output}\n```" elif exit_code == 0: msg = "✅ Hermes update finished successfully." else: msg = "❌ Hermes update failed. Check the gateway logs or run `hermes update` manually for details." - await adapter.send( - chat_id, - msg, - metadata=_non_conversational_metadata(metadata, platform=platform), - ) - logger.info( - "Sent post-update notification to %s:%s (exit=%s)", - platform_str, - chat_id, - exit_code, - ) + await adapter.send(chat_id, msg, metadata=_non_conversational_metadata(metadata, platform=platform)) + logger.info("Sent post-update notification to %s:%s (exit=%s)", platform_str, chat_id, exit_code) except Exception as e: logger.warning("Post-update notification failed: %s", e) finally: if cleanup: - active_pending_path.unlink(missing_ok=True) - claimed_path.unlink(missing_ok=True) - output_path.unlink(missing_ok=True) - exit_code_path.unlink(missing_ok=True) + for p in (active_pending_path, paths.claimed, paths.output, paths.exit_code): + p.unlink(missing_ok=True) return True @@ -781,9 +663,7 @@ class GatewayNotificationsMixin: data = json.loads(notify_path.read_text(encoding="utf-8")) platform_str = data.get("platform") chat_id = data.get("chat_id") - chat_type = data.get("chat_type") thread_id = data.get("thread_id") - message_id = data.get("message_id") if not platform_str or not chat_id: return None @@ -791,10 +671,7 @@ class GatewayNotificationsMixin: platform = Platform(platform_str) transport = resolve_delivery_transport(platform, self.config, self.adapters) if transport is None: - logger.debug( - "Restart notification skipped: no live transport for %s", - platform_str, - ) + logger.debug("Restart notification skipped: no live transport for %s", platform_str) return None platform_cfg = self.config.platforms.get(platform) @@ -806,42 +683,28 @@ class GatewayNotificationsMixin: return None metadata = self._thread_metadata_for_target( - platform, - chat_id, - thread_id, - chat_type=chat_type, - reply_to_message_id=message_id, - adapter=transport.adapter, + platform, chat_id, thread_id, chat_type=data.get("chat_type"), + reply_to_message_id=data.get("message_id"), adapter=transport.adapter, ) if data.get("delivered_via_upstream_relay") is True: metadata = dict(metadata or {}) - if data.get("user_id"): - metadata["user_id"] = str(data["user_id"]) - if data.get("scope_id"): - metadata["scope_id"] = str(data["scope_id"]) + for field in ("user_id", "scope_id"): + if data.get(field): + metadata[field] = str(data[field]) result = await transport.send( - platform, - str(chat_id), - "♻ Gateway restarted successfully. Your session continues.", + platform, str(chat_id), "♻ Gateway restarted successfully. Your session continues.", metadata=_non_conversational_metadata(metadata, platform=platform), ) # adapter.send() catches provider errors (e.g. "Chat not found") and returns - # SendResult(success=False) rather than raising, so inspect the result before claiming - # success — otherwise the log line hides real delivery failures. + # SendResult(success=False) rather than raising, so inspect the result before claiming success. if result is not None and getattr(result, "success", True) is False: logger.warning( "Restart notification to %s:%s was not delivered: %s", - platform_str, - chat_id, - getattr(result, "error", "send returned success=False"), + platform_str, chat_id, getattr(result, "error", "send returned success=False"), ) return None - logger.info( - "Sent restart notification to %s:%s", - platform_str, - chat_id, - ) + logger.info("Sent restart notification to %s:%s", platform_str, chat_id) return str(platform_str), str(chat_id), str(thread_id) if thread_id else None except Exception as e: logger.warning("Restart notification failed: %s", e) @@ -849,31 +712,57 @@ class GatewayNotificationsMixin: finally: notify_path.unlink(missing_ok=True) - async def _send_home_channel_startup_notifications( - self, - *, - skip_targets: Optional[set[tuple[str, str, Optional[str]]]] = None, - ) -> set[tuple[str, str, Optional[str]]]: - """Notify configured home channels that the gateway is back online. - - The notification is best-effort and sent once per connected platform - home channel. ``skip_targets`` lets startup avoid duplicate messages - when a more specific restart notification is queued for the same chat. - """ - from gateway.run import _non_conversational_metadata, resolve_delivery_transport - delivered: set[tuple[str, str, Optional[str]]] = set() - skipped = skip_targets or set() - message = "♻️ Gateway online — Hermes is back and ready." - + def _home_channel_transports(self): + """Yield ``(platform, platform_cfg, home, transport)`` for every home channel with a live transport.""" + from gateway.run import resolve_delivery_transport for platform, platform_cfg in self.config.platforms.items(): home = platform_cfg.home_channel if not home or not home.chat_id: continue - transport = resolve_delivery_transport(platform, self.config, self.adapters) if transport is None: continue + yield platform, platform_cfg, home, transport + async def _send_home_channel_message(self, platform, home, transport, message: str, failure_fmt: str) -> bool: + """Best-effort send to one home channel; True on success, failures logged with ``failure_fmt``.""" + from gateway.run import _non_conversational_metadata + try: + metadata = self._thread_metadata_for_target(platform, home.chat_id, home.thread_id, adapter=transport.adapter) + if transport.is_relay: + metadata = dict(metadata or {}) + if home.user_id: + metadata["user_id"] = home.user_id + if home.scope_id: + metadata["scope_id"] = home.scope_id + send_metadata = _non_conversational_metadata(metadata, platform=platform) + if send_metadata is not None or transport.is_relay: + result = await transport.send(platform, str(home.chat_id), message, metadata=send_metadata) + else: + result = await transport.adapter.send(str(home.chat_id), message) + if result is not None and getattr(result, "success", True) is False: + logger.warning( + failure_fmt, platform.value, home.chat_id, getattr(result, "error", "send returned success=False"), + ) + return False + return True + except Exception as exc: + logger.warning(failure_fmt, platform.value, home.chat_id, exc) + return False + + async def _send_home_channel_startup_notifications( + self, *, skip_targets: Optional[set[tuple[str, str, Optional[str]]]] = None + ) -> set[tuple[str, str, Optional[str]]]: + """Notify configured home channels that the gateway is back online. + + Best-effort, once per connected platform home channel. ``skip_targets`` lets startup avoid + duplicate messages when a more specific restart notification is queued for the same chat. + """ + delivered: set[tuple[str, str, Optional[str]]] = set() + skipped = skip_targets or set() + message = "♻️ Gateway online — Hermes is back and ready." + + for platform, platform_cfg, home, transport in self._home_channel_transports(): if not platform_cfg.gateway_restart_notification: logger.info( "Home-channel startup notification suppressed: %s has gateway_restart_notification=false", @@ -885,51 +774,11 @@ class GatewayNotificationsMixin: if target in skipped or target in delivered: continue - try: - metadata = self._thread_metadata_for_target( - platform, - home.chat_id, - home.thread_id, - adapter=transport.adapter, - ) - if transport.is_relay: - metadata = dict(metadata or {}) - if home.user_id: - metadata["user_id"] = home.user_id - if home.scope_id: - metadata["scope_id"] = home.scope_id - send_metadata = _non_conversational_metadata(metadata, platform=platform) - if send_metadata is not None or transport.is_relay: - result = await transport.send( - platform, - str(home.chat_id), - message, - metadata=send_metadata, - ) - else: - result = await transport.adapter.send(str(home.chat_id), message) - if result is not None and getattr(result, "success", True) is False: - logger.warning( - "Home-channel startup notification failed for %s:%s: %s", - platform.value, - home.chat_id, - getattr(result, "error", "send returned success=False"), - ) - continue - + if await self._send_home_channel_message( + platform, home, transport, message, "Home-channel startup notification failed for %s:%s: %s", + ): delivered.add(target) - logger.info( - "Sent home-channel startup notification to %s:%s", - platform.value, - home.chat_id, - ) - except Exception as exc: - logger.warning( - "Home-channel startup notification failed for %s:%s: %s", - platform.value, - home.chat_id, - exc, - ) + logger.info("Sent home-channel startup notification to %s:%s", platform.value, home.chat_id) return delivered @@ -940,80 +789,31 @@ class GatewayNotificationsMixin: — /resume, /history, and session_search all silently break. Best-effort: failures are logged, not raised. """ - from gateway.run import _non_conversational_metadata, resolve_delivery_transport error = getattr(self, "_session_db_init_error", None) if not error: return from hermes_state import classify_persistence_error, format_session_db_unavailable - - cause = classify_persistence_error(error) - hint = format_session_db_unavailable() - if cause == "corrupt": + if classify_persistence_error(error) == "corrupt": message = ( "⚠️ Session database corruption detected. Messages may not be " - "persisted. Recovery options:\n" - "1. Run `hermes doctor --fix`\n" - "2. Salvage with: sqlite3 ~/.hermes/state.db \".recover\" " - "(then replace state.db)\n" + "persisted. Recovery options:\n1. Run `hermes doctor --fix`\n" + "2. Salvage with: sqlite3 ~/.hermes/state.db \".recover\" (then replace state.db)\n" "3. Restore from a backup in ~/.hermes/backups/\n" "Run `hermes doctor` for sanitized diagnostics." ) else: message = ( f"⚠️ Session database unavailable — messages may not be persisted. " - f"{hint}\n" - f"Run `hermes doctor` for diagnostics." + f"{format_session_db_unavailable()}\nRun `hermes doctor` for diagnostics." ) - logger.warning( - "Broadcasting state.db failure warning to home channels: %s", error - ) + logger.warning("Broadcasting state.db failure warning to home channels: %s", error) - for platform, platform_cfg in self.config.platforms.items(): - home = platform_cfg.home_channel - if not home or not home.chat_id: - continue - transport = resolve_delivery_transport(platform, self.config, self.adapters) - if transport is None: - continue - try: - metadata = self._thread_metadata_for_target( - platform, - home.chat_id, - home.thread_id, - adapter=transport.adapter, - ) - if transport.is_relay: - metadata = dict(metadata or {}) - if home.user_id: - metadata["user_id"] = home.user_id - if home.scope_id: - metadata["scope_id"] = home.scope_id - send_metadata = _non_conversational_metadata(metadata, platform=platform) - if send_metadata is not None or transport.is_relay: - result = await transport.send( - platform, - str(home.chat_id), - message, - metadata=send_metadata, - ) - else: - result = await transport.adapter.send(str(home.chat_id), message) - if result is not None and getattr(result, "success", True) is False: - logger.warning( - "state.db warning notification failed for %s:%s: %s", - platform.value, - home.chat_id, - getattr(result, "error", "send returned success=False"), - ) - except Exception as exc: - logger.warning( - "state.db warning notification failed for %s:%s: %s", - platform.value, - home.chat_id, - exc, - ) + for platform, _platform_cfg, home, transport in self._home_channel_transports(): + await self._send_home_channel_message( + platform, home, transport, message, "state.db warning notification failed for %s:%s: %s", + ) def _build_process_event_source(self, evt: dict): """Resolve the canonical source for a synthetic background-process event. @@ -1021,12 +821,8 @@ class GatewayNotificationsMixin: Prefer the persisted session-store origin; the active foreground event causes cross-topic bleed. """ from gateway.run import _parse_session_key - from gateway.session import SessionSource - session_key = str(evt.get("session_key") or "").strip() - derived_platform = "" - derived_chat_type = "" - derived_chat_id = "" + derived = {} if session_key: try: @@ -1035,32 +831,22 @@ class GatewayNotificationsMixin: if entry and getattr(entry, "origin", None): return entry.origin except Exception as exc: - logger.debug( - "Synthetic process-event session-store lookup failed for %s: %s", - session_key, - exc, - ) + logger.debug("Synthetic process-event session-store lookup failed for %s: %s", session_key, exc) cached_source = self._get_cached_session_source(session_key) if cached_source is not None: return cached_source - _parsed = _parse_session_key(session_key) - if _parsed: - derived_platform = _parsed["platform"] - derived_chat_type = _parsed["chat_type"] - derived_chat_id = _parsed["chat_id"] + derived = _parse_session_key(session_key) or {} - platform_name = str(evt.get("platform") or derived_platform or "").strip().lower() - chat_type = str(evt.get("chat_type") or derived_chat_type or "").strip().lower() - chat_id = str(evt.get("chat_id") or derived_chat_id or "").strip() + platform_name = str(evt.get("platform") or derived.get("platform") or "").strip().lower() + chat_type = str(evt.get("chat_type") or derived.get("chat_type") or "").strip().lower() + chat_id = str(evt.get("chat_id") or derived.get("chat_id") or "").strip() if not platform_name or not chat_type or not chat_id: logger.warning( "Synthetic event source unresolvable: " - "session_key=%r platform=%r chat_type=%r chat_id=%r " - "evt_type=%s", - session_key, platform_name, chat_type, chat_id, - evt.get("type", "?"), + "session_key=%r platform=%r chat_type=%r chat_id=%r evt_type=%s", + session_key, platform_name, chat_type, chat_id, evt.get("type", "?"), ) return None @@ -1076,18 +862,18 @@ class GatewayNotificationsMixin: except Exception: raise ValueError(platform_name) except Exception: - logger.warning( - "Synthetic process event has invalid platform metadata: %r", - platform_name, - ) + logger.warning("Synthetic process event has invalid platform metadata: %r", platform_name) return None - scope_id = str(evt.get("scope_id") or "").strip() or None + def _opt(field: str) -> Optional[str]: + return str(evt.get(field) or "").strip() or None + + scope_id = _opt("scope_id") if scope_id is None and chat_type not in ("dm", "thread"): # Reconstructed (non-persisted) source for a scoped chat with no scope discriminator: a - # relay connector's fail-closed tenant guard may decline the reply unless user_id resolves it - # (resolveByUser). Don't fail — DMs/author-bound chats still route and native adapters need - # no scope_id — but warn so a post-restart egress decline isn't silent. + # relay connector's fail-closed tenant guard may decline the reply unless user_id resolves + # it. Don't fail — DMs/author-bound chats still route and native adapters need no + # scope_id — but warn so a post-restart egress decline isn't silent. logger.warning( "Synthetic event source for %s chat=%s (%s) reconstructed " "without scope_id; scoped relay egress may be declined by " @@ -1095,13 +881,8 @@ class GatewayNotificationsMixin: platform_name, chat_id, chat_type, ) return SessionSource( - platform=platform, - chat_id=chat_id, - chat_type=chat_type, - thread_id=str(evt.get("thread_id") or "").strip() or None, - user_id=str(evt.get("user_id") or "").strip() or None, - user_name=str(evt.get("user_name") or "").strip() or None, - scope_id=scope_id, + platform=platform, chat_id=chat_id, chat_type=chat_type, thread_id=_opt("thread_id"), + user_id=_opt("user_id"), user_name=_opt("user_name"), scope_id=scope_id, ) async def _drain_watch_notifications(self, completion_queue) -> None: @@ -1124,16 +905,68 @@ class GatewayNotificationsMixin: except Exception as exc: logger.error("Watch notification injection error: %s", exc) - async def _inject_watch_notification( - self, synth_text: str, evt: dict, - ) -> Optional[bool]: + def _adapter_by_platform_value(self, platform_name: str): + """Literal ``p.value == platform_name`` scan over connected adapters (native adapters only).""" + for p, a in self.adapters.items(): + if p.value == platform_name: + return a + return None + + async def _self_post_api_server(self, adapter, synth_text: str, raw_sid: str, evt: dict) -> bool: + """Deliver to a non-push (api_server) session by raw session id. + + Async-delegation completions are persisted as a durable delivery row — after the parent + turn's event.complete the CLIENT owns the next turn on this stateless surface, so never + self-post them as a new role=user prompt. Other watch events wake the session via self-post. + """ + from gateway.wake import deliver_wake, persist_delegation_delivery + if evt.get("type") == "async_delegation": + try: + logger.info( + "Async delegation completion — persisting delivery row for api_server session %s (no wake turn)", + raw_sid, + ) + await persist_delegation_delivery(adapter, text=synth_text, session_id=raw_sid, evt=evt) + return True + except Exception as e: + logger.warning("Async delegation delivery persist failed for session %s: %s", raw_sid, e) + return False + try: + logger.info("Watch pattern notification — waking api_server session %s via self-post", raw_sid) + await deliver_wake(adapter, text=synth_text, session_id=raw_sid) + return True + except Exception as e: + logger.warning("Watch notification self-post wake failed for session %s: %s", raw_sid, e) + return False + + def _resolve_injection_adapter(self, platform_name: str): + """Adapter for a synthetic-event platform: transport resolver first, literal scan as fallback. + + Alias-aware resolution (relay-plane): one adapter under Platform.RELAY fronts N logical + platforms, so a literal ``p.value == platform_name`` scan misses "slack" and drops the + completion as "no gateway route". Native adapter wins; relay is eligible only when it + advertises fronting the logical platform. The legacy literal scan is still correct for + native adapters and keeps minimal runner stubs (tests) / exotic platform strings working + when the resolver can't run. + """ + from gateway.run import resolve_delivery_transport + try: + _transport = resolve_delivery_transport(Platform(platform_name), self.config, self.adapters) + except Exception: + _transport = None + if _transport is not None: + return _transport.adapter + return self._adapter_by_platform_value(platform_name) + + async def _inject_watch_notification(self, synth_text: str, evt: dict) -> Optional[bool]: """Inject a watch/completion notification as a synthetic message event. Routing comes from the queued event, never the active foreground message. Returns ``True`` on adapter acceptance, ``False`` on retryable adapter failure, ``None`` with no gateway route. Not transactional: a crash after acceptance can replay (at-least-once). """ - from gateway.run import _parse_session_key, resolve_delivery_transport + from gateway.run import _parse_session_key + from gateway.wake import adapter_supports_push source = await asyncio.to_thread(self._build_process_event_source, evt) if not source: # API-server sessions bind the RAW X-Hermes-Session-Id key (_bind_api_server_session), not a @@ -1145,53 +978,10 @@ class GatewayNotificationsMixin: raw_sid = _sk if raw_sid: adapter = self.adapters.get(Platform.API_SERVER) - from gateway.wake import ( - adapter_supports_push, - deliver_wake, - persist_delegation_delivery, - ) if adapter is not None and not adapter_supports_push(adapter): - if evt.get("type") == "async_delegation": - # after the parent turn's event.complete the CLIENT owns the next turn on - # this stateless surface. Persist the completion as a durable delivery row — - # never self-post it as a new role=user prompt. - try: - logger.info( - "Async delegation completion — persisting " - "delivery row for api_server session %s " - "(no wake turn)", - raw_sid, - ) - await persist_delegation_delivery( - adapter, text=synth_text, - session_id=raw_sid, evt=evt, - ) - return True - except Exception as e: - logger.warning( - "Async delegation delivery persist failed " - "for session %s: %s", - raw_sid, e, - ) - return False - try: - logger.info( - "Watch pattern notification — waking api_server " - "session %s via self-post", - raw_sid, - ) - await deliver_wake(adapter, text=synth_text, session_id=raw_sid) - return True - except Exception as e: - logger.warning( - "Watch notification self-post wake failed for " - "session %s: %s", - raw_sid, e, - ) - return False + return await self._self_post_api_server(adapter, synth_text, raw_sid, evt) logger.warning( - "Dropping watch notification for raw session %s: no " - "api_server adapter to self-post through", + "Dropping watch notification for raw session %s: no api_server adapter to self-post through", raw_sid, ) return None @@ -1201,93 +991,27 @@ class GatewayNotificationsMixin: ) return None platform_name = source.platform.value if hasattr(source.platform, "value") else str(source.platform) - # Alias-aware resolution (relay-plane): one adapter under Platform.RELAY fronts N logical - # platforms, so a literal ``p.value == platform_name`` scan misses "slack" and drops the - # completion as "no gateway route". Use the shared transport resolver — native adapter wins; - # relay is eligible only when it advertises fronting the logical platform. - adapter = None - try: - _platform_enum = Platform(platform_name) - except (ValueError, KeyError): - _platform_enum = None - if _platform_enum is not None: - try: - _transport = resolve_delivery_transport( - _platform_enum, self.config, self.adapters, - ) - except Exception: - _transport = None - if _transport is not None: - adapter = _transport.adapter - if adapter is None: - # Legacy literal scan — still correct for native adapters; keeps minimal runner stubs (tests) - # and exotic platform strings working when the resolver can't run. - for p, a in self.adapters.items(): - if p.value == platform_name: - adapter = a - break + adapter = self._resolve_injection_adapter(platform_name) if not adapter: return None - from gateway.wake import adapter_supports_push as _wake_push_ok - if not _wake_push_ok(adapter): + if not adapter_supports_push(adapter): # Non-push adapter (api_server) resolved WITH routing metadata: its chat_id is the raw session # id (_bind_api_server_session binds chat_id = session_id), so handle_message would run the # wake under a build_session_key() key that never matches the raw session — self-post. - from gateway.wake import deliver_wake, persist_delegation_delivery raw_sid = str(evt.get("origin_session_id") or "").strip() or str(source.chat_id or "") - if evt.get("type") == "async_delegation": - # Same client-owns-the-turn rule as the raw-key branch above: persist the completion as a - # delivery row, never self-post it as a new role=user prompt. - try: - logger.info( - "Async delegation completion — persisting delivery " - "row for api_server session %s (no wake turn)", - raw_sid, - ) - await persist_delegation_delivery( - adapter, text=synth_text, session_id=raw_sid, evt=evt, - ) - return True - except Exception as e: - logger.warning( - "Async delegation delivery persist failed for " - "session %s: %s", - raw_sid, e, - ) - return False - try: - logger.info( - "Watch pattern notification — waking api_server session " - "%s via self-post", - raw_sid, - ) - await deliver_wake(adapter, text=synth_text, session_id=raw_sid) - return True - except Exception as e: - logger.warning( - "Watch notification self-post wake failed for session " - "%s: %s", - raw_sid, e, - ) - return False + return await self._self_post_api_server(adapter, synth_text, raw_sid, evt) try: metadata = {} parent_session_id = str(evt.get("parent_session_id") or "").strip() if parent_session_id: metadata["gateway_session_id"] = parent_session_id synth_event = MessageEvent( - text=synth_text, - message_type=MessageType.TEXT, - source=source, - internal=True, - message_id=str(evt.get("message_id") or "").strip() or None, - metadata=metadata, + text=synth_text, message_type=MessageType.TEXT, source=source, internal=True, + message_id=str(evt.get("message_id") or "").strip() or None, metadata=metadata, ) logger.info( "Watch pattern notification — injecting for %s chat=%s thread=%s", - platform_name, - source.chat_id, - source.thread_id, + platform_name, source.chat_id, source.thread_id, ) # Relay-plane egress priming: a synthetic turn injected right after a restart reaches a relay # adapter whose per-chat routing caches are cold (they warm only on inbound), so its replies @@ -1320,6 +1044,28 @@ class GatewayNotificationsMixin: return (evt_type, producer_id, started_at) return None + def _mark_completions_delivered_locked(self, identities) -> None: + """Move identities inflight -> delivered and trim retention. Caller holds ``_completion_delivery_lock``.""" + for identity in identities: + self._completion_deliveries_inflight.discard(identity) + self._completion_deliveries_delivered[identity] = None + while len(self._completion_deliveries_delivered) > self._completion_delivery_retention: + self._completion_deliveries_delivered.popitem(last=False) + + def _completion_identity_seen(self, identity, *, claim: bool = False) -> bool: + """True when ``identity`` is inflight or already delivered this gateway lifecycle. + + With ``claim`` an unseen identity is atomically marked inflight (same lock hold). + """ + with self._completion_delivery_lock: + seen = ( + identity in self._completion_deliveries_inflight + or identity in self._completion_deliveries_delivered + ) + if claim and not seen: + self._completion_deliveries_inflight.add(identity) + return seen + async def _classify_completion_target(self, parent_session_id: str) -> str: """Classify an async-completion delivery target before adapter acceptance. @@ -1337,10 +1083,7 @@ class GatewayNotificationsMixin: try: parent = await session_db.get_session(parent_session_id) except Exception: - logger.debug( - "Async-completion pre-flight parent lookup failed for %s", - parent_session_id, exc_info=True, - ) + logger.debug("Async-completion pre-flight parent lookup failed for %s", parent_session_id, exc_info=True) return "retry" if parent is None: return "terminal" @@ -1348,126 +1091,104 @@ class GatewayNotificationsMixin: return "deliver" end_reason = str(parent.get("end_reason") or "") if end_reason != "compression": - # An ended parent is unreachable only when the USER closed the thread of work (/new -> - # session_reset / new_session, user_exit, session_switch). Idle/timeout ends are normal on - # scale-to-zero relays — the chat stays routable and the resolver retargets, so dropping loses - # finished work. Boundary set shared with the resolver (_USER_BOUNDARY_END_REASONS): no drift. - if end_reason in _USER_BOUNDARY_END_REASONS: - return "terminal" - return "deliver" + # Only a USER-closed thread of work (/new, user_exit, session_switch) is unreachable; + # idle/timeout ends are normal on scale-to-zero relays — the chat stays routable and the + # resolver retargets. Boundary set shared with the resolver: no drift. + return "terminal" if end_reason in _USER_BOUNDARY_END_REASONS else "deliver" try: tip_session_id = await session_db.get_compression_tip(parent_session_id) if not tip_session_id or tip_session_id == parent_session_id: - # Rotation caught mid-flight: parent is compression-ended but - # its continuation isn't visible yet. Retry, don't drop. + # Rotation caught mid-flight: parent is compression-ended but its continuation isn't + # visible yet. Retry, don't drop. return "retry" tip = await session_db.get_session(tip_session_id) except Exception: - logger.debug( - "Async-completion pre-flight tip lookup failed for %s", - parent_session_id, exc_info=True, - ) + logger.debug("Async-completion pre-flight tip lookup failed for %s", parent_session_id, exc_info=True) return "retry" if tip is None or tip.get("ended_at"): return "retry" return "deliver" - async def _deliver_completion_notification( - self, synth_text: str, evt: dict, - ) -> Optional[bool]: + @staticmethod + def _settle_durable_claim(kind: str, delegation_id: str, claim_id: str) -> None: + """Best-effort ``drop``/``release`` of a durable completion claim.""" + fn_name, fail_msg = _DURABLE_CLAIM_OPS[kind] + try: + import tools.async_delegation as _ad + getattr(_ad, fn_name)(delegation_id, claim_id) + except Exception: + logger.debug(fail_msg, exc_info=True) + + async def _preflight_completion_delivery(self, evt: dict) -> "_CompletionClaim": + """Claim the durable row (async delegations) and verify the target before adapter acceptance. + + Adapter acceptance is not proof of delivery: the inner resolver can still fail closed inside + the pipeline after acceptance, falsely acking the durable row. Verifying first gives drops an + honest durable disposition. + """ + claim = self._CompletionClaim() + evt_type = evt.get("type") + if evt_type == "async_delegation": + claim.delegation_id = str(evt.get("delegation_id") or "") + if claim.delegation_id: + try: + from tools.async_delegation import claim_completion_delivery + claim.claim_id = f"gateway:{id(self)}:{__import__('uuid').uuid4().hex}" + if not claim_completion_delivery(claim.delegation_id, claim.claim_id): + claim.proceed = False + return claim + except Exception as exc: + logger.warning("Could not claim durable async completion %s: %s", claim.delegation_id, exc) + claim.proceed, claim.early_result = False, False + return claim + elif evt_type != "completion": + return claim + + # Background completions carry only session_key, so after /new the OLD session's notification + # would land in the chat's NEW session. Stamped events get the same pre-flight as async + # delegations; unstamped ones deliver. + parent_session_id = str(evt.get("parent_session_id") or "").strip() + if not parent_session_id: + return claim + verdict = await self._classify_completion_target(parent_session_id) + if verdict == "terminal": + if evt_type == "async_delegation": + logger.warning( + "Async delegation %s targets permanently-gone session %s; " + "terminally dropping delivery (result remains in the delegation records).", + claim.delegation_id or "", parent_session_id, + ) + if claim.claim_id: + self._settle_durable_claim("drop", claim.delegation_id, claim.claim_id) + else: + logger.warning( + "Background process %s completion targets " + "permanently-gone session %s (user boundary such as " + "/new); dropping notification (output remains " + "available via process(action='log')).", + evt.get("session_id") or "", parent_session_id, + ) + claim.proceed = False + elif verdict == "retry": + # Transient uncertainty (session DB down / compression rotation mid-flight): tell the + # watcher to re-poll and retry rather than drop or misroute the result. + if claim.claim_id: + self._settle_durable_claim("release", claim.delegation_id, claim.claim_id) + claim.proceed, claim.early_result = False, False + return claim + + async def _deliver_completion_notification(self, synth_text: str, evt: dict) -> Optional[bool]: """Deliver once per live gateway, or return False for a retry. ``True``: adapter accepted; ``False``: injection failed, claim released for retry; ``None``: another same-lifecycle caller owns/delivered it, or no route. No cross-process exactly-once. """ identity = self._completion_delivery_identity(evt) - durable_claim_id = "" - durable_delegation_id = "" - if evt.get("type") == "async_delegation": - durable_delegation_id = str(evt.get("delegation_id") or "") - if durable_delegation_id: - try: - from tools.async_delegation import claim_completion_delivery - - durable_claim_id = f"gateway:{id(self)}:{__import__('uuid').uuid4().hex}" - if not claim_completion_delivery( - durable_delegation_id, durable_claim_id, - ): - return None - except Exception as exc: - logger.warning( - "Could not claim durable async completion %s: %s", - durable_delegation_id, exc, - ) - return False - parent_session_id = str(evt.get("parent_session_id") or "").strip() - if parent_session_id: - # Adapter acceptance is not proof of delivery: the inner resolver can still fail closed - # inside the pipeline after acceptance, falsely acking the durable row as delivered. - # Verify the target before acceptance so drops get an honest durable disposition. - verdict = await self._classify_completion_target(parent_session_id) - if verdict == "terminal": - logger.warning( - "Async delegation %s targets permanently-gone session %s; " - "terminally dropping delivery (result remains in the " - "delegation records).", - durable_delegation_id or "", parent_session_id, - ) - if durable_claim_id: - try: - from tools.async_delegation import drop_completion_delivery - - drop_completion_delivery( - durable_delegation_id, durable_claim_id, - ) - except Exception: - logger.debug( - "Could not drop durable completion claim", - exc_info=True, - ) - return None - if verdict == "retry": - if durable_claim_id: - try: - from tools.async_delegation import release_completion_delivery - - release_completion_delivery( - durable_delegation_id, durable_claim_id, - ) - except Exception: - logger.debug( - "Could not release durable completion claim", - exc_info=True, - ) - return False - elif evt.get("type") == "completion": - # Background completions carry only session_key, so after /new the OLD session's - # notification would land in the chat's NEW session. Stamped events get the same - # pre-flight as async delegations (_classify_completion_target); unstamped ones deliver. - parent_session_id = str(evt.get("parent_session_id") or "").strip() - if parent_session_id: - verdict = await self._classify_completion_target(parent_session_id) - if verdict == "terminal": - logger.warning( - "Background process %s completion targets " - "permanently-gone session %s (user boundary such as " - "/new); dropping notification (output remains " - "available via process(action='log')).", - evt.get("session_id") or "", parent_session_id, - ) - return None - if verdict == "retry": - # Transient uncertainty (session DB down / compression rotation mid-flight): tell the - # watcher to re-poll and retry rather than drop or misroute the result. - return False - if identity is not None: - with self._completion_delivery_lock: - if ( - identity in self._completion_deliveries_inflight - or identity in self._completion_deliveries_delivered - ): - return None - self._completion_deliveries_inflight.add(identity) + claim = await self._preflight_completion_delivery(evt) + if not claim.proceed: + return claim.early_result + if identity is not None and self._completion_identity_seen(identity, claim=True): + return None accepted = False try: @@ -1478,54 +1199,27 @@ class GatewayNotificationsMixin: if identity is not None: with self._completion_delivery_lock: - self._completion_deliveries_inflight.discard(identity) - self._completion_deliveries_delivered[identity] = None - while ( - len(self._completion_deliveries_delivered) - > self._completion_delivery_retention - ): - self._completion_deliveries_delivered.popitem(last=False) + self._mark_completions_delivered_locked((identity,)) - # When the durable async-delegation producer branch is present, its SQLite row is the - # authoritative replay state — ack it after adapter acceptance; no parallel ledger here. - if durable_claim_id: + # The durable async-delegation row is the authoritative replay state — ack it after + # adapter acceptance; no parallel ledger here. + if claim.claim_id: try: from tools.async_delegation import complete_completion_delivery - - complete_completion_delivery( - durable_delegation_id, durable_claim_id, - ) + complete_completion_delivery(claim.delegation_id, claim.claim_id) except Exception as exc: - logger.warning( - "Could not acknowledge durable async completion %s: %s", - durable_delegation_id, exc, - ) + logger.warning("Could not acknowledge durable async completion %s: %s", claim.delegation_id, exc) return True finally: if identity is not None and not accepted: with self._completion_delivery_lock: self._completion_deliveries_inflight.discard(identity) - if durable_claim_id and not accepted: - try: - from tools.async_delegation import release_completion_delivery - - release_completion_delivery( - durable_delegation_id, durable_claim_id, - ) - except Exception: - logger.debug("Could not release durable completion claim", exc_info=True) + if claim.claim_id and not accepted: + self._settle_durable_claim("release", claim.delegation_id, claim.claim_id) @staticmethod - def _completion_notification_batch_key(evt: dict) -> tuple[str, ...]: - """Return a routing-complete key for short-window process fan-in.""" - return tuple(str(evt.get(field) or "") for field in ( - "session_key", - "platform", - "chat_type", - "chat_id", - "thread_id", - "user_id", - )) + def _event_route_key(evt: dict, fields: tuple[str, ...]) -> tuple[str, ...]: + return tuple(str(evt.get(field) or "") for field in fields) @staticmethod def _format_coalesced_process_completions(entries: list[tuple[str, dict, asyncio.Future]]) -> str: @@ -1544,14 +1238,10 @@ class GatewayNotificationsMixin: # Completion output normally passes the terminal redactor at the producer seam, but that is # configurable and this is user-facing, so keep the unconditional gateway floor. Redact # BEFORE slicing: truncating first can leave a credential fragment the patterns miss. - output = _redact_gateway_user_facing_secrets( - str(evt.get("output") or "") - ).strip() + output = _redact_gateway_user_facing_secrets(str(evt.get("output") or "")).strip() if len(output) > 800: output = f"[… truncated …]\n{output[-800:]}" - lines.append( - f"\n- {session_id}: exit_code={exit_code}, reason={reason}" - ) + lines.append(f"\n- {session_id}: exit_code={exit_code}, reason={reason}") if output: lines.append(output) omitted = len(entries) - len(shown) @@ -1560,25 +1250,14 @@ class GatewayNotificationsMixin: f"\n- … and {omitted} more completion(s); inspect them with " "the process tool if they affect the conclusion." ) - lines.append( - "If a result does not change the current conclusion, absorb it silently.]" - ) + lines.append("If a result does not change the current conclusion, absorb it silently.]") return "\n".join(lines) def _record_coalesced_completion_siblings(self, events: list[dict]) -> None: """Extend a successful primary delivery claim to its batched siblings.""" + identities = [i for i in map(self._completion_delivery_identity, events) if i is not None] with self._completion_delivery_lock: - for evt in events: - identity = self._completion_delivery_identity(evt) - if identity is None: - continue - self._completion_deliveries_inflight.discard(identity) - self._completion_deliveries_delivered[identity] = None - while ( - len(self._completion_deliveries_delivered) - > self._completion_delivery_retention - ): - self._completion_deliveries_delivered.popitem(last=False) + self._mark_completions_delivered_locked(identities) async def _flush_process_completion_batch(self, key: tuple[str, ...]) -> None: """Deliver one short-window completion batch and resolve its waiters.""" @@ -1588,30 +1267,23 @@ class GatewayNotificationsMixin: try: await asyncio.sleep(self._completion_notification_batch_window) entries = self._completion_notification_batches.pop(key, []) - # Detach before adapter delivery. A completion that arrives while - # this batch is in flight must be able to schedule the next flush. + # Detach before adapter delivery: a completion arriving while this batch is in flight + # must be able to schedule the next flush. if self._completion_notification_batch_tasks.get(key) is current_task: self._completion_notification_batch_tasks.pop(key, None) if not entries: return - if len(entries) == 1: - synth_text = entries[0][0] - else: - synth_text = self._format_coalesced_process_completions(entries) + synth_text = entries[0][0] if len(entries) == 1 else self._format_coalesced_process_completions(entries) # A duplicate primary can legitimately return None from the lifecycle dedupe seam; try the # next batch identity so a fresh sibling is never discarded with that duplicate. delivered = None for _text, candidate_evt, _future in entries: - delivered = await self._deliver_completion_notification( - synth_text, candidate_evt, - ) + delivered = await self._deliver_completion_notification(synth_text, candidate_evt) if delivered is not None: break if delivered is True and len(entries) > 1: - self._record_coalesced_completion_siblings( - [evt for _text, evt, _future in entries] - ) + self._record_coalesced_completion_siblings([evt for _text, evt, _future in entries]) except asyncio.CancelledError: # Shutdown may cancel us mid fan-in or while adapter delivery is blocked: recover entries not # yet detached and resolve every waiter as retryable before adapters are torn down. @@ -1637,9 +1309,7 @@ class GatewayNotificationsMixin: self._completion_notification_batches_stopping = True tasks = { task - for task in getattr( - self, "_completion_notification_batch_flush_tasks", set() - ) + for task in getattr(self, "_completion_notification_batch_flush_tasks", set()) if not task.done() } for task in tasks: @@ -1657,46 +1327,35 @@ class GatewayNotificationsMixin: getattr(self, "_completion_notification_batch_tasks", {}).clear() getattr(self, "_completion_notification_batch_flush_tasks", set()).clear() - async def _enqueue_process_completion_notification( - self, synth_text: str, evt: dict, - ) -> Optional[bool]: + async def _enqueue_process_completion_notification(self, synth_text: str, evt: dict) -> Optional[bool]: """Fan in concurrent process completions that share one conversation.""" # Some unit tests construct GatewayRunner with object.__new__. Keep the # batching seam lazy so those focused lifecycle tests remain valid. - if not hasattr(self, "_completion_notification_batches"): - self._completion_notification_batches = {} - if not hasattr(self, "_completion_notification_batch_tasks"): - self._completion_notification_batch_tasks = {} - if not hasattr(self, "_completion_notification_batch_flush_tasks"): - self._completion_notification_batch_flush_tasks = set() - if not hasattr(self, "_completion_notification_batch_window"): - self._completion_notification_batch_window = 0.1 - if not hasattr(self, "_completion_notification_batches_stopping"): - self._completion_notification_batches_stopping = False + for attr, default in ( + ("_completion_notification_batches", dict), + ("_completion_notification_batch_tasks", dict), + ("_completion_notification_batch_flush_tasks", set), + ("_completion_notification_batch_window", lambda: 0.1), + ("_completion_notification_batches_stopping", lambda: False), + ("_background_tasks", set), + ): + if not hasattr(self, attr): + setattr(self, attr, default()) if self._completion_notification_batches_stopping: return False - key = self._completion_notification_batch_key(evt) + key = self._event_route_key(evt, self._COMPLETION_BATCH_KEY_FIELDS) future = asyncio.get_running_loop().create_future() - self._completion_notification_batches.setdefault(key, []).append( - (synth_text, evt, future) - ) + self._completion_notification_batches.setdefault(key, []).append((synth_text, evt, future)) if key not in self._completion_notification_batch_tasks: - task = asyncio.create_task( - self._flush_process_completion_batch(key) - ) + task = asyncio.create_task(self._flush_process_completion_batch(key)) self._completion_notification_batch_tasks[key] = task - # Keep the flush alive under the gateway's normal lifecycle accounting; runners built via - # object.__new__ (focused tests) lazily receive the same ownership set. - if not hasattr(self, "_background_tasks"): - self._background_tasks = set() + # Keep the flush alive under the gateway's normal lifecycle accounting. self._background_tasks.add(task) self._completion_notification_batch_flush_tasks.add(task) task.add_done_callback(self._background_tasks.discard) - task.add_done_callback( - self._completion_notification_batch_flush_tasks.discard - ) + task.add_done_callback(self._completion_notification_batch_flush_tasks.discard) return await future def _enrich_async_delegation_routing(self, evt: dict) -> None: @@ -1718,33 +1377,14 @@ class GatewayNotificationsMixin: evt["thread_id"] = parsed["thread_id"] @staticmethod - def _async_delegation_group_key(evt: dict) -> tuple[str, ...]: - """Return the async-completion coalescing key: originating session, parent session, route.""" - return tuple(str(evt.get(field) or "") for field in ( - "session_key", - "parent_session_id", - "platform", - "chat_type", - "chat_id", - "thread_id", - "user_id", - )) + def _settle_sibling_claims(siblings: list[tuple[dict, str]], fn, fail_msg: str) -> None: + for evt, claim_id in siblings: + try: + fn(evt, claim_id) + except Exception: + logger.debug(fail_msg, exc_info=True) - @staticmethod - def _format_coalesced_async_delegations(blocks: list[str]) -> str: - """Join per-delegation formatted blocks into one consolidated turn.""" - header = ( - f"[IMPORTANT: {len(blocks)} background subagent delegations " - "completed for this session. Treat these results as one " - "completion batch and send at most one consolidated user-facing " - "response. If a result does not change the current conclusion, " - "absorb it silently.]" - ) - return "\n\n".join([header, *blocks]) - - async def _deliver_async_delegation_group( - self, group: list[dict], - ) -> Optional[bool]: + async def _deliver_async_delegation_group(self, group: list[dict]) -> Optional[bool]: """Deliver a same-session batch of async completions as ONE turn. Single-event groups ride the per-event path. Multi-event groups deliver the primary via @@ -1755,20 +1395,14 @@ class GatewayNotificationsMixin: """ from gateway.run import _format_gateway_process_notification from tools.process_registry import process_registry as _pr - deliverable: list[tuple[dict, str]] = [] for evt in group: synth_text = _format_gateway_process_notification(evt) if not synth_text: continue identity = self._completion_delivery_identity(evt) - if identity is not None: - with self._completion_delivery_lock: - if ( - identity in self._completion_deliveries_inflight - or identity in self._completion_deliveries_delivered - ): - continue + if identity is not None and self._completion_identity_seen(identity): + continue deliverable.append((evt, synth_text)) if not deliverable: @@ -1777,62 +1411,45 @@ class GatewayNotificationsMixin: evt, synth_text = deliverable[0] return await self._deliver_completion_notification(synth_text, evt) - from tools.async_delegation import ( - claim_event_delivery, - complete_event_delivery, - release_event_delivery, - ) - + from tools.async_delegation import claim_event_delivery, complete_event_delivery, release_event_delivery primary_evt, primary_text = deliverable[0] blocks = [primary_text] siblings: list[tuple[dict, str]] = [] for evt, synth_text in deliverable[1:]: claim_id = claim_event_delivery(evt, f"gateway-batch:{id(self)}") if claim_id is None: - # Another consumer owns this row's delivery; keep its result - # out of our consolidated text so it is never double-injected. + # Another consumer owns this row's delivery; keep its result out of our + # consolidated text so it is never double-injected. continue siblings.append((evt, claim_id)) blocks.append(synth_text) if not siblings: - return await self._deliver_completion_notification( - primary_text, primary_evt, - ) + return await self._deliver_completion_notification(primary_text, primary_evt) - consolidated = self._format_coalesced_async_delegations(blocks) + header = ( + f"[IMPORTANT: {len(blocks)} background subagent delegations " + "completed for this session. Treat these results as one " + "completion batch and send at most one consolidated user-facing " + "response. If a result does not change the current conclusion, absorb it silently.]" + ) + consolidated = "\n\n".join([header, *blocks]) delivered: Optional[bool] = False try: - delivered = await self._deliver_completion_notification( - consolidated, primary_evt, - ) + delivered = await self._deliver_completion_notification(consolidated, primary_evt) finally: if delivered is True: - for evt, claim_id in siblings: - try: - complete_event_delivery(evt, claim_id) - except Exception: - logger.debug( - "Could not acknowledge coalesced durable completion", - exc_info=True, - ) - self._record_coalesced_completion_siblings( - [evt for evt, _claim_id in siblings] + self._settle_sibling_claims( + siblings, complete_event_delivery, "Could not acknowledge coalesced durable completion", ) + self._record_coalesced_completion_siblings([evt for evt, _claim_id in siblings]) else: - # Not delivered — release every sibling claim so a retry or another consumer can claim it, - # honestly leaving the durable rows pending. - for evt, claim_id in siblings: - try: - release_event_delivery(evt, claim_id) - except Exception: - logger.debug( - "Could not release coalesced durable claim", - exc_info=True, - ) + # Not delivered — release every sibling claim so a retry or another consumer can claim + # it, honestly leaving the durable rows pending. + self._settle_sibling_claims(siblings, release_event_delivery, "Could not release coalesced durable claim") if delivered is None: - # The primary was dropped/owned elsewhere but the siblings - # still need delivery — requeue just them for the next tick. + # The primary was dropped/owned elsewhere but the siblings still need delivery — + # requeue just them for the next tick. for evt, _claim_id in siblings: _pr.completion_queue.put(evt) return delivered @@ -1856,26 +1473,17 @@ class GatewayNotificationsMixin: evt = _pr.completion_queue.get_nowait() except Exception: break - if evt.get("type") == "async_delegation": - async_events.append(evt) - else: - requeue.append(evt) + (async_events if evt.get("type") == "async_delegation" else requeue).append(evt) for evt in requeue: _pr.completion_queue.put(evt) # A same-tick drain often carries several completions for the SAME session (a fan-out # finishing together); delivering each individually floods it with N synthetic turns. # Group by full gateway route + parent session: one consolidated turn per group. groups: dict[tuple[str, ...], list[dict]] = {} - group_order: list[tuple[str, ...]] = [] for evt in async_events: self._enrich_async_delegation_routing(evt) - key = self._async_delegation_group_key(evt) - if key not in groups: - groups[key] = [] - group_order.append(key) - groups[key].append(evt) - for key in group_order: - group = groups[key] + groups.setdefault(self._event_route_key(evt, self._ASYNC_GROUP_KEY_FIELDS), []).append(evt) + for group in groups.values(): try: delivered = await self._deliver_async_delegation_group(group) if delivered is False: @@ -1889,6 +1497,84 @@ class GatewayNotificationsMixin: logger.debug("Async delegation watcher error: %s", e) await asyncio.sleep(interval) + @staticmethod + def _redacted_output_tail(session, limit: int) -> str: + """Last ``limit`` chars of process output through the secret redactors (unconditional floor).""" + from gateway.run import _redact_gateway_user_facing_secrets + new_output = session.output_buffer[-limit:] if session.output_buffer else "" + if new_output: + from agent.redact import redact_terminal_output + new_output = redact_terminal_output(new_output, getattr(session, "command", "") or "") + # redact_terminal_output() is unforced, so it returns raw text when security.redact_secrets + # is off. This goes straight to the platform adapter, so it needs the same unconditional + # floor as agent-notify. + new_output = _redact_gateway_user_facing_secrets(new_output) + return new_output + + async def _send_watcher_message(self, platform_name: str, chat_id, thread_id, message_text: str) -> None: + from gateway.run import _non_conversational_metadata + adapter = self._adapter_by_platform_value(platform_name) + if adapter and chat_id: + try: + send_meta = {"thread_id": thread_id} if thread_id else None + await adapter.send( + chat_id, message_text, metadata=_non_conversational_metadata(send_meta, platform=platform_name), + ) + except Exception as e: + logger.error("Watcher delivery error: %s", e) + + @staticmethod + def _build_process_completion_event(watcher: dict, session, session_id: str) -> dict: + """Build the synthetic ``completion`` event for an agent-notify watcher.""" + from gateway.run import _redact_gateway_user_facing_secrets + from agent.redact import redact_terminal_output + from tools.ansi_strip import strip_ansi + _command = getattr(session, "command", "") or "" + _raw = strip_ansi(session.output_buffer) if session.output_buffer else "" + _raw = redact_terminal_output(_raw, _command) + # Truncate on line boundaries (never start mid-line): keep the last ~2000 chars snapped to + # the preceding newline, prepending a marker when output was cut. + _LIMIT = 2000 + if len(_raw) > _LIMIT: + _tail = _raw[-_LIMIT:] + _nl = _tail.find("\n") + _tail = _tail[_nl + 1:] if _nl != -1 else _tail + _out = f"[… output truncated — showing last {len(_tail)} chars]\n{_tail}" + else: + _out = _raw + return { + "type": "completion", + "session_id": session_id, + **{k: watcher.get(k, "") for k in _WATCHER_ROUTE_FIELDS}, + "message_id": str(watcher.get("message_id") or "").strip() or None, + "started_at": getattr(session, "started_at", None), + "command": _redact_gateway_user_facing_secrets(_command), + "exit_code": session.exit_code, + "completion_reason": getattr(session, "completion_reason", "exited"), + "termination_source": getattr(session, "termination_source", ""), + "output": _redact_gateway_user_facing_secrets(_out), + # Spawning conversation's session-db id (stamped in terminal_tool); lets delivery + # pre-flight drop this completion if the user closed that session (/new) first. + "parent_session_id": ( + watcher.get("parent_session_id") or getattr(session, "parent_session_id", "") or "" + ), + } + + def _format_process_final_message(self, session_id: str, session, notify_mode: str) -> str: + from gateway.run import _format_concise_process_notification, _redact_gateway_user_facing_secrets + new_output = self._redacted_output_tail(session, 1000) + if notify_mode != "concise": + return ( + f"[Background process {session_id} finished with exit code {session.exit_code}~ " + f"Here's the final output:\n{new_output}]" + ) + _started = getattr(session, "started_at", None) + _dur = max(0.0, time.time() - _started) if isinstance(_started, (int, float)) else None + return _format_concise_process_notification( + session_id, _redact_gateway_user_facing_secrets(getattr(session, "command", "") or ""), + session.exit_code, new_output, duration_seconds=_dur, + ) + async def _run_process_watcher(self, watcher: dict) -> None: """Periodically check a background process and push updates to the user. @@ -1897,22 +1583,12 @@ class GatewayNotificationsMixin: concise (default, one-line; failures append output tail) / all (running updates + final raw output) / result (final raw only) / error (final raw only if exit != 0) / off. """ - from gateway.run import ( - _format_concise_process_notification, - _non_conversational_metadata, - _redact_gateway_user_facing_secrets, - ) - from tools.process_registry import process_registry - + from tools.process_registry import format_process_notification, process_registry session_id = watcher["session_id"] interval = watcher["check_interval"] - session_key = watcher.get("session_key", "") platform_name = watcher.get("platform", "") chat_id = watcher.get("chat_id", "") thread_id = watcher.get("thread_id", "") - user_id = watcher.get("user_id", "") - user_name = watcher.get("user_name", "") - message_id = str(watcher.get("message_id") or "").strip() or None agent_notify = watcher.get("notify_on_complete", False) notify_mode = self._load_background_notifications_mode() @@ -1920,8 +1596,7 @@ class GatewayNotificationsMixin: session_id, interval, notify_mode, agent_notify) if notify_mode == "off" and not agent_notify: - # Still wait for the process to exit so we can log it, but don't - # push any messages to the user. + # Still wait for the process to exit so we can log it, but don't push any messages. while True: await asyncio.sleep(interval) session = process_registry.get(session_id) @@ -1946,153 +1621,43 @@ class GatewayNotificationsMixin: # Agent-triggered completion: inject a synthetic message unless the agent already consumed # the result via wait/log. poll() is read-only and deliberately does NOT mark consumed — # a status check must not suppress this delivery turn. - from tools.process_registry import format_process_notification, process_registry as _pr_check - if agent_notify and not _pr_check.is_completion_consumed(session_id): - from agent.redact import redact_terminal_output - from tools.ansi_strip import strip_ansi - _command = getattr(session, "command", "") or "" - _raw = strip_ansi(session.output_buffer) if session.output_buffer else "" - _raw = redact_terminal_output(_raw, _command) - _command = _redact_gateway_user_facing_secrets(_command) - # Truncate on line boundaries (never start mid-line): keep the last ~2000 chars - # snapped to the preceding newline, prepending a marker when output was cut. - _LIMIT = 2000 - if len(_raw) > _LIMIT: - _tail = _raw[-_LIMIT:] - _nl = _tail.find("\n") - _tail = _tail[_nl + 1:] if _nl != -1 else _tail - _out = f"[… output truncated — showing last {len(_tail)} chars]\n{_tail}" - else: - _out = _raw - _out = _redact_gateway_user_facing_secrets(_out) - completion_evt = { - "type": "completion", - "session_id": session_id, - "session_key": session_key, - "platform": platform_name, - "chat_type": watcher.get("chat_type", ""), - "chat_id": chat_id, - "thread_id": thread_id, - "user_id": user_id, - "user_name": user_name, - "message_id": message_id, - "started_at": getattr(session, "started_at", None), - "command": _command, - "exit_code": session.exit_code, - "completion_reason": getattr(session, "completion_reason", "exited"), - "termination_source": getattr(session, "termination_source", ""), - "output": _out, - # Spawning conversation's session-db id (stamped in terminal_tool); lets delivery - # pre-flight drop this completion if the user closed that session (/new) first. - "parent_session_id": ( - watcher.get("parent_session_id") - or getattr(session, "parent_session_id", "") - or "" - ), - } + if agent_notify and not process_registry.is_completion_consumed(session_id): + completion_evt = self._build_process_completion_event(watcher, session, session_id) synth_text = format_process_notification(completion_evt) if not synth_text: break - delivered = await self._enqueue_process_completion_notification( - synth_text, completion_evt, - ) + delivered = await self._enqueue_process_completion_notification(synth_text, completion_evt) if delivered is False: - # The process remains terminal; retry after failed - # adapter injection instead of suppressing the result. + # The process remains terminal; retry after failed adapter injection instead + # of suppressing the result. continue break # Normal text-only notification. Skip when the agent already consumed this completion via # wait/log (output returned inline) — the raw "finished" message would be a duplicate. # The agent_notify skip FALLS THROUGH here, hence this check. poll() is read-only. - if _pr_check.is_completion_consumed(session_id): + if process_registry.is_completion_consumed(session_id): logger.debug( "Process watcher: completion for %s already consumed " - "via wait/log — skipping raw notification (#65379)", - session_id, + "via wait/log — skipping raw notification (#65379)", session_id, ) break - # Decide whether to notify based on mode should_notify = ( notify_mode in {"concise", "all", "result"} or (notify_mode == "error" and session.exit_code not in {0, None}) ) if should_notify: - new_output = session.output_buffer[-1000:] if session.output_buffer else "" - if new_output: - from agent.redact import redact_terminal_output - new_output = redact_terminal_output( - new_output, getattr(session, "command", "") or "" - ) - # redact_terminal_output() is unforced, so it returns raw text when - # security.redact_secrets is off. This goes straight to the platform - # adapter, so it needs the same unconditional floor as agent-notify. - new_output = _redact_gateway_user_facing_secrets(new_output) - if notify_mode == "concise": - _cmd_disp = _redact_gateway_user_facing_secrets( - getattr(session, "command", "") or "" - ) - _started = getattr(session, "started_at", None) - _dur = None - if isinstance(_started, (int, float)): - _dur = max(0.0, time.time() - _started) - message_text = _format_concise_process_notification( - session_id, - _cmd_disp, - session.exit_code, - new_output, - duration_seconds=_dur, - ) - else: - message_text = ( - f"[Background process {session_id} finished with exit code {session.exit_code}~ " - f"Here's the final output:\n{new_output}]" - ) - adapter = None - for p, a in self.adapters.items(): - if p.value == platform_name: - adapter = a - break - if adapter and chat_id: - try: - send_meta = {"thread_id": thread_id} if thread_id else None - await adapter.send( - chat_id, - message_text, - metadata=_non_conversational_metadata(send_meta, platform=platform_name), - ) - except Exception as e: - logger.error("Watcher delivery error: %s", e) + message_text = self._format_process_final_message(session_id, session, notify_mode) + await self._send_watcher_message(platform_name, chat_id, thread_id, message_text) break elif has_new_output and notify_mode == "all" and not agent_notify: - # New output available -- deliver status update (only in "all" mode) - # Skip periodic updates for agent_notify watchers (they only care about completion) - new_output = session.output_buffer[-500:] if session.output_buffer else "" - if new_output: - from agent.redact import redact_terminal_output - new_output = redact_terminal_output( - new_output, getattr(session, "command", "") or "" - ) - new_output = _redact_gateway_user_facing_secrets(new_output) + # New output — deliver a status update (only in "all" mode; agent_notify watchers + # only care about completion). + new_output = self._redacted_output_tail(session, 500) message_text = ( - f"[Background process {session_id} is still running~ " - f"New output:\n{new_output}]" + f"[Background process {session_id} is still running~ New output:\n{new_output}]" ) - adapter = None - for p, a in self.adapters.items(): - if p.value == platform_name: - adapter = a - break - if adapter and chat_id: - try: - send_meta = {"thread_id": thread_id} if thread_id else None - await adapter.send( - chat_id, - message_text, - metadata=_non_conversational_metadata(send_meta, platform=platform_name), - ) - except Exception as e: - logger.error("Watcher delivery error: %s", e) + await self._send_watcher_message(platform_name, chat_id, thread_id, message_text) logger.debug("Process watcher ended: %s", session_id) diff --git a/gateway/run_shutdown.py b/gateway/run_shutdown.py index b37646bbfb..7a012b9f6d 100644 --- a/gateway/run_shutdown.py +++ b/gateway/run_shutdown.py @@ -7,25 +7,25 @@ so ``patch("gateway.run.X")`` keeps intercepting them at call time. from __future__ import annotations -import logging -from typing import TYPE_CHECKING import asyncio +import dataclasses +import json +import logging import os import shlex import sys import threading import time from contextlib import suppress +from pathlib import Path +from typing import TYPE_CHECKING, Any, Callable, Dict, Optional + from gateway.config import Platform from gateway.restart import ( - DEFAULT_GATEWAY_CRON_DRAIN_TIMEOUT, - GATEWAY_SERVICE_RESTART_EXIT_CODE, - resolve_cron_drain_budget, + DEFAULT_GATEWAY_CRON_DRAIN_TIMEOUT, GATEWAY_SERVICE_RESTART_EXIT_CODE, resolve_cron_drain_budget ) from gateway.run_common import _UNSET from gateway.shutdown_watchdog import arm_shutdown_watchdog, resolve_shutdown_watchdog_delay -from pathlib import Path -from typing import Any, Callable, Dict, Optional, Tuple if TYPE_CHECKING: # string annotations only; never imported at runtime (cycle) from gateway.run import GatewayRunner, TurnRunner # noqa: F401 @@ -33,10 +33,78 @@ if TYPE_CHECKING: # string annotations only; never imported at runtime (cycle) # Log-record parity with the origin module. logger = logging.getLogger("gateway.run") +# Windows has no bash/setsid chain: a tiny detached Python watcher waits for the gateway PID to +# exit (bounded), then spawns ``hermes gateway restart``. +_WINDOWS_RESTART_WATCHER = """ +import os, subprocess, sys, time +from hermes_cli._subprocess_compat import windows_detach_flags_without_breakaway +pid = int(sys.argv[1]) +restart_after_s = float(sys.argv[2]) +cmd = sys.argv[3:] +deadline = time.monotonic() + restart_after_s + +def _alive(p): + # On Windows, os.kill(pid, 0) is NOT a no-op — it maps to + # GenerateConsoleCtrlEvent(0, pid) (bpo-14484). Use the + # Win32 handle-based existence check instead. + if os.name == 'nt': + import ctypes + k32 = ctypes.windll.kernel32 + k32.OpenProcess.restype = ctypes.c_void_p + k32.WaitForSingleObject.restype = ctypes.c_uint + k32.GetLastError.restype = ctypes.c_uint + h = k32.OpenProcess(0x1000 | 0x100000, False, int(p)) + if not h: + return k32.GetLastError() != 87 + try: + return k32.WaitForSingleObject(h, 0) == 0x102 + finally: + k32.CloseHandle(h) + try: + os.kill(int(p), 0) + return True + except ProcessLookupError: + return False + except PermissionError: + return True + except OSError: + return False + +while time.monotonic() < deadline: + if not _alive(pid): + break + time.sleep(0.2) +subprocess.Popen( + cmd, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + creationflags=windows_detach_flags_without_breakaway(), +) +""".strip() + + +def _send_failed(result: Any) -> bool: + """True when an adapter ``send()`` result explicitly reports failure.""" + return result is not None and getattr(result, "success", True) is False + class GatewayShutdownMixin: """Stop/drain/restart, scale-to-zero and active-work accounting methods for GatewayRunner.""" + @dataclasses.dataclass + class _StopContext: + """State threaded through the ``_stop_*`` phases of one ``stop()`` run.""" + + deferred_count: Callable[[], int] + started_at: Optional[float] = None + active_agents: dict = dataclasses.field(default_factory=dict) + timed_out: bool = False + drain_elapsed: float = 0.0 + + def elapsed(self) -> float: + return time.monotonic() - self.started_at + + # Active-work accounting def _active_work_count(self) -> int: """All agent work the gateway must expose and drain as one total.""" return ( @@ -47,81 +115,58 @@ class GatewayShutdownMixin: ) def _active_cron_job_count(self) -> int: - """Count of cron jobs currently executing (``cron.scheduler._running_job_ids``). - - Cron jobs run on the scheduler's own thread pool, outside ``self._running_agents`` which - every OTHER active-work check reads; without this the shutdown drain can kill a cron job's - tool subprocess mid-run. Best-effort: returns 0 if the cron module can't be imported. - """ + """Cron jobs currently executing — they run outside ``_running_agents``; 0 if cron can't import.""" try: from cron.scheduler import get_running_job_ids return len(get_running_job_ids()) except Exception: return 0 - def _active_api_run_count(self) -> int: - """Count API-server work that is outside ``_running_agents``. + def _api_server_hook(self, name: str, *args: Any) -> int: + """Call the primary API-server adapter's ``name`` hook, clamped >= 0 (0 when the hook is absent). - Only the primary API server owns the HTTP listener (secondary multiplex profiles cannot - bind a port), so only the primary registry is a source of this work. + Only the primary API server owns the HTTP listener, so only it is a source of this work. """ + adapter = getattr(self, "adapters", {}).get(Platform.API_SERVER) + helper = getattr(adapter, name, None) + return max(0, int(helper(*args))) if callable(helper) else 0 + + def _active_api_run_count(self) -> int: + """API-server work that is outside ``_running_agents``.""" try: - adapter = getattr(self, "adapters", {}).get(Platform.API_SERVER) - helper = getattr(adapter, "active_agent_work_count", None) - return max(0, int(helper())) if callable(helper) else 0 + return self._api_server_hook("active_agent_work_count") except Exception: return 0 def _interrupt_api_server_runs(self, reason: str) -> int: - """Interrupt API-server agents that are not in ``_running_agents``. - - Counterpart of ``_active_api_run_count()``: must reach the same agents when the drain times - out. Duck-typed so an adapter (or test double) without the hook is skipped, not raised on. - """ + """Interrupt API-server agents not in ``_running_agents`` (same set ``_active_api_run_count`` counts).""" try: - adapter = getattr(self, "adapters", {}).get(Platform.API_SERVER) - helper = getattr(adapter, "interrupt_active_runs", None) - return max(0, int(helper(reason))) if callable(helper) else 0 + return self._api_server_hook("interrupt_active_runs", reason) except Exception as exc: logger.debug("Failed interrupting api_server runs during shutdown: %s", exc) return 0 def _active_deferred_agent_worker_count(self) -> int: - """Count executor workers that outlived their owning gateway turn. - - A timed-out hygiene compression keeps running in its executor thread. - Some paths defer agent cleanup; the live Codex path keeps its cached - agent. In both cases the turn can finish before the worker does, so - ``_running_agents`` no longer represents it. Count the worker itself. - """ + """Executor workers that outlived their gateway turn (e.g. a timed-out hygiene compression).""" workers = getattr(self, "_deferred_agent_workers", None) if not isinstance(workers, dict): return 0 return sum(1 for future in list(workers) if not future.done()) - def _track_deferred_agent_worker( - self, - future: asyncio.Future, - agent: Any, - ) -> None: + def _track_deferred_agent_worker(self, future: asyncio.Future, agent: Any) -> None: """Expose an executor worker to drain/interrupt until it really exits.""" workers = getattr(self, "_deferred_agent_workers", None) if workers is None: - workers = {} - self._deferred_agent_workers = workers + workers = self._deferred_agent_workers = {} workers[future] = agent def _discard_worker(done_future: asyncio.Future) -> None: workers.pop(done_future, None) - # Some tracked workers intentionally outlive the coroutine that - # started them and therefore have no later waiter. Consume their - # terminal exception so asyncio does not emit an unhandled-future - # warning after the worker eventually unwinds (#98973). + # Workers that outlive their starting coroutine have no later waiter: consume the + # terminal exception so asyncio emits no unhandled-future warning. if not done_future.cancelled(): - try: + with suppress(Exception): done_future.exception() - except Exception: - pass future.add_done_callback(_discard_worker) @@ -141,20 +186,15 @@ class GatewayShutdownMixin: request_hard_interrupt(agent, reason) interrupted += 1 except Exception as exc: - logger.debug( - "Failed interrupting deferred agent worker during shutdown: %s", - exc, - ) + logger.debug("Failed interrupting deferred agent worker during shutdown: %s", exc) return interrupted + # Scale-to-zero idle detection / dormant-quiesce def _scale_to_zero_has_live_background_work(self) -> bool: - """Live background work that must block a suspend. + """Live background work (delegations, processes, pending watchers) that must block a suspend. - Backgrounded delegate_task / kanban / terminal(background=true) are NOT counted by - _running_agent_count() but suspending loses them; checks tracked tasks + process registry + - pending completion watchers. PERMANENT supervised watchers (_hermes_supervised_watcher) are - excluded — they live for the whole process (including the scale-to-zero watcher itself), so - counting them would make this True forever and the gateway could never go dormant. + PERMANENT supervised watchers (_hermes_supervised_watcher, incl. the scale-to-zero watcher + itself) are excluded, else this would be True forever and the gateway could never go dormant. """ if any( not t.done() and not getattr(t, "_hermes_supervised_watcher", False) @@ -163,179 +203,135 @@ class GatewayShutdownMixin: return True try: from tools.async_delegation import active_count - if active_count() > 0: return True except Exception: # noqa: BLE001 - never let the idle check raise logger.debug("scale-to-zero async-delegation check failed", exc_info=True) try: from tools.process_registry import process_registry - - if process_registry.has_any_active(): - return True - if process_registry.pending_watchers: + if process_registry.has_any_active() or process_registry.pending_watchers: return True except Exception: # noqa: BLE001 - never let the idle check raise logger.debug("scale-to-zero bg-work check failed", exc_info=True) return False - def _scale_to_zero_idle_timeout_seconds(self) -> float: + @staticmethod + def _gateway_cfg_section(name: str) -> Optional[dict]: + """``gateway.`` from the user config when it is a dict, else None (never raises).""" from gateway.run import _load_gateway_config - from gateway.scale_to_zero import parse_idle_timeout_seconds - - raw = None try: user_cfg = _load_gateway_config() gw = user_cfg.get("gateway") if isinstance(user_cfg, dict) else None - stz = gw.get("scale_to_zero") if isinstance(gw, dict) else None - if isinstance(stz, dict): - raw = stz.get("idle_timeout_minutes") + section = gw.get(name) if isinstance(gw, dict) else None except Exception: # noqa: BLE001 - raw = None - return parse_idle_timeout_seconds(raw) + return None + return section if isinstance(section, dict) else None + + def _scale_to_zero_idle_timeout_seconds(self) -> float: + from gateway.scale_to_zero import parse_idle_timeout_seconds + stz = self._gateway_cfg_section("scale_to_zero") + return parse_idle_timeout_seconds(stz.get("idle_timeout_minutes") if stz else None) def _restart_loop_guard_config(self) -> tuple: - """Return ``(max_restarts, window_seconds, max_gap_seconds)`` for the auto-resume - restart-loop breaker, from ``gateway.restart_loop_guard`` with module defaults as fallback. + """``(max_restarts, window_seconds, max_gap_seconds)`` for the restart-loop breaker. - ``max_restarts <= 0`` disables the breaker. ``max_gap_seconds`` is the longest spacing - between consecutive restart-interrupted boots that still counts as the same loop, so a - crash cycle slower than ``window_seconds`` stays visible. + ``max_restarts <= 0`` disables it; ``max_gap_seconds`` is the longest spacing between + restart-interrupted boots that still counts as one loop. """ - from gateway.run import _load_gateway_config from gateway import restart_loop_guard as _rlg + rlg = self._gateway_cfg_section("restart_loop_guard") or {} - max_restarts = _rlg.DEFAULT_MAX_RESTARTS - window_seconds = _rlg.DEFAULT_WINDOW_SECONDS - max_gap_seconds = _rlg.DEFAULT_MAX_GAP_SECONDS - try: - user_cfg = _load_gateway_config() - gw = user_cfg.get("gateway") if isinstance(user_cfg, dict) else None - rlg = gw.get("restart_loop_guard") if isinstance(gw, dict) else None - if isinstance(rlg, dict): - if isinstance(rlg.get("max_restarts"), int): - max_restarts = rlg["max_restarts"] - if isinstance(rlg.get("window_seconds"), int) and rlg["window_seconds"] > 0: - window_seconds = rlg["window_seconds"] - if ( - isinstance(rlg.get("max_gap_seconds"), int) - and rlg["max_gap_seconds"] > 0 - ): - max_gap_seconds = rlg["max_gap_seconds"] - except Exception: # noqa: BLE001 - pass - return max_restarts, window_seconds, max_gap_seconds + def _int_or(key: str, default: int, positive: bool) -> int: + value = rlg.get(key) + if isinstance(value, int) and (value > 0 or not positive): + return value + return default + + return ( + _int_or("max_restarts", _rlg.DEFAULT_MAX_RESTARTS, positive=False), + _int_or("window_seconds", _rlg.DEFAULT_WINDOW_SECONDS, positive=True), + _int_or("max_gap_seconds", _rlg.DEFAULT_MAX_GAP_SECONDS, positive=True), + ) def _scale_to_zero_active_messaging_platforms(self) -> list: - """ENABLED platforms that count for the relay-only arm gate. + """ENABLED MESSAGING platforms for the relay-only arm gate. - Two load-bearing filters: enabled only (config.platforms is pre-seeded with disabled - placeholders for the whole catalog) and MESSAGING only (the api_server is a loopback listener - force-enabled on every hosted container with no outbound socket; counting it silently - disarmed the feature everywhere). Mirrors the non-messaging exclusion in _connect_platforms. + config.platforms is pre-seeded with disabled placeholders, and the api_server is force-enabled + on every hosted container (counting it silently disarmed the feature everywhere). """ if not self.config: return [] non_messaging = {Platform.LOCAL, Platform.API_SERVER, Platform.WEBHOOK} try: return [ - p - for p, pc in self.config.platforms.items() + p for p, pc in self.config.platforms.items() if getattr(pc, "enabled", False) and p not in non_messaging ] except Exception: # noqa: BLE001 return [] + @staticmethod + def _relay_wake_url_or_none(): + from gateway.relay import relay_wake_url + try: + return relay_wake_url() + except Exception: # noqa: BLE001 + return None + def _scale_to_zero_should_arm(self) -> bool: """Whether to start the idle watcher (D1/D11/§3.4(1)).""" - from gateway.relay import relay_wake_url - from gateway.scale_to_zero import ( - messaging_is_relay_only_or_absent, - scale_to_zero_enabled, - should_arm, - ) - - platforms = self._scale_to_zero_active_messaging_platforms() - try: - wake_url = relay_wake_url() - except Exception: # noqa: BLE001 - wake_url = None + from gateway.scale_to_zero import messaging_is_relay_only_or_absent, scale_to_zero_enabled, should_arm return should_arm( enabled=scale_to_zero_enabled(), - relay_only_or_absent=messaging_is_relay_only_or_absent(platforms), - wake_url=wake_url, + relay_only_or_absent=messaging_is_relay_only_or_absent(self._scale_to_zero_active_messaging_platforms()), + wake_url=self._relay_wake_url_or_none(), ) def _log_scale_to_zero_not_armed_reason(self) -> None: - """Log why the idle watcher did NOT arm — but only for an OPTED-IN instance. - - A non-opted instance (no HERMES_SCALE_TO_ZERO stamp) not arming is normal and stays silent; - with the stamp set, the surprise earns one INFO line so the answer is a log grep. - """ - from gateway.relay import relay_wake_url - from gateway.scale_to_zero import ( - messaging_is_relay_only_or_absent, - scale_to_zero_enabled, - ) - + """One INFO line on why the idle watcher did NOT arm — only for an OPTED-IN instance.""" + from gateway.scale_to_zero import messaging_is_relay_only_or_absent, scale_to_zero_enabled try: - enabled = scale_to_zero_enabled() - if not enabled: + if not scale_to_zero_enabled(): return # not opted in — normal, stay quiet - active = [ - getattr(p, "value", p) - for p in self._scale_to_zero_active_messaging_platforms() - ] - relay_only = messaging_is_relay_only_or_absent(active) - try: - wake_url = relay_wake_url() - except Exception: # noqa: BLE001 - wake_url = None + active = [getattr(p, "value", p) for p in self._scale_to_zero_active_messaging_platforms()] logger.info( "scale-to-zero: NOT armed despite opt-in — " "relay_only_or_absent=%s (enabled platforms=%s), wake_url=%s. " "Need relay-only messaging + a registered wake URL.", - relay_only, - active or "none", - "set" if wake_url else "MISSING", + messaging_is_relay_only_or_absent(active), active or "none", + "set" if self._relay_wake_url_or_none() else "MISSING", ) except Exception: # noqa: BLE001 - diagnostics must never block startup logger.debug("scale-to-zero: not-armed reason logging failed", exc_info=True) def _scale_to_zero_is_idle(self) -> bool: from gateway.scale_to_zero import is_idle + # FULL work aggregate (cron + API runs live outside _running_agents) with fail-AWAKE reads: + # the drain counters swallow errors to 0, which for a suspend predicate would look idle, so + # an unreadable source counts as work here. - # The FULL work aggregate, not _running_agent_count(): cron jobs and API-server runs live - # outside _running_agents, so counting agents alone let a suspend land mid-cron-job. - # Fail-AWAKE accounting: the shutdown-drain counters swallow exceptions to 0, which is fine - # for a drain but unsafe for a suspend predicate (a transient read failure would look idle). - # Here an unreadable source counts as work (sentinel 1) so the machine stays awake. - try: + def _read_or_awake(label: str, fn: Callable[[], Any], busy_sentinel: Any) -> Any: + try: + return fn() + except Exception: # noqa: BLE001 - unreadable source => assume busy + logger.debug("scale-to-zero: %s unreadable — staying awake", label, exc_info=True) + return busy_sentinel + + def _cron_count() -> int: from cron.scheduler import get_running_job_ids + return len(get_running_job_ids()) - cron_count = len(get_running_job_ids()) - except Exception: # noqa: BLE001 - unreadable source => assume busy - logger.debug("scale-to-zero: cron work count unreadable — staying awake", exc_info=True) - cron_count = 1 - try: - adapter = getattr(self, "adapters", {}).get(Platform.API_SERVER) - helper = getattr(adapter, "active_agent_work_count", None) - api_count = max(0, int(helper())) if callable(helper) else 0 - except Exception: # noqa: BLE001 - unreadable source => assume busy - logger.debug("scale-to-zero: api work count unreadable — staying awake", exc_info=True) - api_count = 1 - # An attached dashboard/desktop/TUI client is inbound activity too; it lives in the DASHBOARD - # process and reaches us as a file mtime refreshed on every WS frame (gateway/scale_to_zero.py). - # Folded into the inbound clock rather than a conjunct: same idle_timeout grace after - # disconnect as a chat message, and a lingering marker cannot pin the box. - last_inbound = self._last_inbound_at - try: + def _dashboard_seen(): from gateway.scale_to_zero import dashboard_client_last_seen + return dashboard_client_last_seen() - seen = dashboard_client_last_seen() - except Exception: # noqa: BLE001 - unreadable source => assume busy - logger.debug("scale-to-zero: dashboard heartbeat unreadable — staying awake", exc_info=True) - seen = time.time() + cron_count = _read_or_awake("cron work count", _cron_count, 1) + api_count = _read_or_awake("api work count", lambda: self._api_server_hook("active_agent_work_count"), 1) + # An attached dashboard/desktop/TUI client (heartbeat file mtime from the dashboard process) is + # inbound activity too — folded into the inbound clock, not a conjunct, so disconnect gets the + # same idle_timeout grace as a message and a lingering marker cannot pin the box. + last_inbound = self._last_inbound_at + seen = _read_or_awake("dashboard heartbeat", _dashboard_seen, time.time()) if seen is not None and seen > last_inbound: last_inbound = seen return is_idle( @@ -346,39 +342,34 @@ class GatewayShutdownMixin: ) def _scale_to_zero_note_real_inbound(self) -> None: - """Stamp real inbound and restore lifecycle after a dormant wake. + """Stamp real inbound and flip status back to running after a dormant wake. - Dormancy marks status `draining` but is not the stop/restart drain: the process stays alive - and should present as running once real traffic wakes it. Internal completion/replay events - deliberately do not call this, so they don't keep an idle gateway awake. + Internal completion/replay events deliberately do not call this (they must not keep an idle + gateway awake). """ self._last_inbound_at = time.time() if getattr(self, "_scale_to_zero_cooldown_until", 0.0) > 0: - try: - self._update_runtime_status("running") - except Exception: # noqa: BLE001 - status restoration is best-effort - logger.debug("scale-to-zero: status restore failed", exc_info=True) + self._scale_to_zero_status("running", "scale-to-zero: status restore failed") self._scale_to_zero_cooldown_until = 0.0 + def _scale_to_zero_status(self, state: str, fail_msg: str) -> None: + """Best-effort runtime status write; failures are debug-logged with ``fail_msg``.""" + try: + self._update_runtime_status(state) + except Exception: # noqa: BLE001 - status is best-effort + logger.debug(fail_msg, exc_info=True) + def _relay_adapter_for_dormancy(self): """Return the connected RELAY adapter, if any (the one go_dormant targets).""" - try: - from gateway.platforms.base import Platform - except Exception: # noqa: BLE001 - return None return self.adapters.get(Platform.RELAY) async def _scale_to_zero_watcher(self, interval: float = 30.0) -> None: """Watch for idle, drive the relay dormant, then self-suspend the machine. - Armed ONLY via _scale_to_zero_should_arm() (HERMES_SCALE_TO_ZERO stamp + relay-only/absent - messaging + wakeUrl). On sustained idle: mark status `draining` (NOT _running=False), relay - adapter.go_dormant() (supervisor-preserving socket close, NOT disconnect()), NO - mark_resume_pending (suspend preserves RAM), THEN suspend via the local flaps socket. The - gateway owns the suspend because Fly autostop sees only INBOUND connections and would freeze - mid-job or before the relay flip (machines run autostop:"off"); autostart stays platform-side. - A re-arm cooldown keeps a wake's drained backlog from being re-quiesced. Off-Fly (no flaps - socket) the watcher does not quiesce at all. + On sustained idle: status `draining` (NOT _running=False), relay go_dormant() (socket close, + NOT disconnect()), no mark_resume_pending (suspend preserves RAM), THEN suspend via the flaps + socket. The gateway owns the suspend because Fly autostop sees only INBOUND connections and + would freeze mid-job or before the relay flip. Off-Fly the watcher does not quiesce at all. """ await asyncio.sleep(min(interval, 30.0)) # let startup settle while self._running: @@ -386,21 +377,14 @@ class GatewayShutdownMixin: await asyncio.sleep(interval) if not self._running: return - if time.time() < self._scale_to_zero_cooldown_until: + if time.time() < self._scale_to_zero_cooldown_until or not self._scale_to_zero_is_idle(): continue - if not self._scale_to_zero_is_idle(): - continue - adapter = self._relay_adapter_for_dormancy() - if adapter is None: - continue - go_dormant = getattr(adapter, "go_dormant", None) + go_dormant = getattr(self._relay_adapter_for_dormancy(), "go_dormant", None) if not callable(go_dormant): continue - # Quiesce only when a suspend can follow. Off-Fly the platform owns the freeze and - # go_dormant()'s socket close arms the reconnect supervisor (re-dial ~1.4s, unflipped - # at freeze, inbound dropped not buffered); stay connected, orphan detection adopts it. + # Quiesce only when a suspend can follow: off-Fly the platform owns the freeze and a + # go_dormant() socket close would just re-dial (~1.4s) unflipped, dropping inbound. from gateway.scale_to_zero import self_suspend_available - if not self_suspend_available(): if not self._scale_to_zero_no_suspend_logged: self._scale_to_zero_no_suspend_logged = True @@ -415,10 +399,7 @@ class GatewayShutdownMixin: "(relay buffered, socket closed) then self-suspending", self._scale_to_zero_idle_timeout_seconds(), ) - try: - self._update_runtime_status("draining") - except Exception: # noqa: BLE001 - status is best-effort - logger.debug("scale-to-zero: status mark failed", exc_info=True) + self._scale_to_zero_status("draining", "scale-to-zero: status mark failed") dormant_ok = True try: result = go_dormant() @@ -436,9 +417,7 @@ class GatewayShutdownMixin: if not dormant_ok: continue if not self._scale_to_zero_is_idle(): - logger.info( - "scale-to-zero: inbound arrived during quiesce — skipping suspend" - ) + logger.info("scale-to-zero: inbound arrived during quiesce — skipping suspend") continue await self._scale_to_zero_self_suspend() except asyncio.CancelledError: @@ -447,22 +426,15 @@ class GatewayShutdownMixin: logger.debug("scale-to-zero watcher iteration error", exc_info=True) async def _scale_to_zero_self_suspend(self) -> None: - """Suspend this Fly machine via the local flaps socket (fail-awake). - - Blocking unix-socket call runs in a worker thread so the loop stays live until the kernel - freeze; nothing meaningful runs until wake. Off-Fly this is a silent no-op. - """ + """Suspend this Fly machine via the local flaps socket (fail-awake; off-Fly a silent no-op).""" from gateway.scale_to_zero import self_suspend_available, suspend_self - try: if not self_suspend_available(): logger.debug( - "scale-to-zero: flaps socket / machine identity absent — " - "dormant without platform suspend" + "scale-to-zero: flaps socket / machine identity absent — dormant without platform suspend" ) return - accepted = await asyncio.to_thread(suspend_self) - if not accepted: + if not await asyncio.to_thread(suspend_self): logger.warning( "scale-to-zero: self-suspend not accepted — machine stays " "awake (fail-awake); will retry on the next idle window" @@ -470,18 +442,11 @@ class GatewayShutdownMixin: except Exception: # noqa: BLE001 - suspend is best-effort, never crash logger.debug("scale-to-zero: self-suspend failed", exc_info=True) - # ------------------------------------------------------------------ - # External drain control (NAS-driven quiesce-without-restart). The dashboard's - # begin/cancel-drain endpoint writes/removes the ``.drain_request.json`` marker - # (gateway/drain_control.py); this watcher flips the gateway between accepting and refusing - # NEW turns WITHOUT exiting. Reversible: NAS begins drain, polls /api/status until - # active_agents hits 0, acts; on cancel/abort the marker is removed and turns resume. - # ------------------------------------------------------------------ + # External drain control: the dashboard's begin/cancel-drain endpoint writes/removes the + # ``.drain_request.json`` marker (gateway/drain_control.py); the watcher flips the gateway + # between accepting and refusing NEW turns WITHOUT exiting (reversible on marker removal). def _enter_external_drain(self) -> None: - """Begin external drain: refuse NEW turns (in-flight ones are NOT interrupted), flip state. - - Idempotent: re-entry only re-writes status. - """ + """Begin external drain: refuse NEW turns (in-flight ones are NOT interrupted). Idempotent.""" if self._external_drain_active: return self._external_drain_active = True @@ -495,17 +460,11 @@ class GatewayShutdownMixin: self._update_runtime_status("draining") def _exit_external_drain(self) -> None: - """Cancel external drain: revert state, re-accept new turns. - - Idempotent. Reverts to ``running`` only when actually mid-drain AND not shutting down — - a real shutdown ``_draining`` must win; never resurrect a stopping gateway. - """ + """Cancel external drain: re-accept new turns. Idempotent; never resurrects a stopping gateway.""" if not self._external_drain_active: return self._external_drain_active = False if self._draining or not self._running: - # A shutdown drain is in progress / the loop has stopped — do not - # clobber the terminal state back to running. logger.info( "External drain marker cleared during shutdown — not reverting " "to running (shutdown takes precedence)." @@ -518,14 +477,8 @@ class GatewayShutdownMixin: self._update_runtime_status("running") async def _drain_control_watcher(self, interval: float = 1.0) -> None: - """Background task: reconcile gateway accept-state with the drain marker. - - Polls ``.drain_request.json`` (presence-based) at 1s: present -> enter drain, absent -> exit; - reconciles once at startup. A marker from a PRIOR instantiation epoch (survived a machine - restart) is treated as absent. Best-effort: tick errors are logged and the loop continues. - """ + """Poll ``.drain_request.json`` at 1s: present -> enter drain, absent -> exit; a stale epoch = absent.""" from gateway.drain_control import drain_requested - while self._running: try: # drain_requested() does a synchronous read_text() on the marker file: at 1s cadence @@ -545,16 +498,11 @@ class GatewayShutdownMixin: await asyncio.sleep(interval) def _update_platform_runtime_status( - self, - platform: str, - *, - platform_state: Optional[str] = None, - error_code: Optional[str] = None, - error_message: Optional[str] = None, - needs_attention: Optional[bool] = None, - retrying_since: Any = _UNSET, + self, platform: str, *, platform_state: Optional[str] = None, + error_code: Optional[str] = None, error_message: Optional[str] = None, + needs_attention: Optional[bool] = None, retrying_since: Any = _UNSET, ) -> None: - try: + with suppress(Exception): from gateway.status import write_runtime_status extra: Dict[str, Any] = {} if needs_attention is not None: @@ -562,59 +510,35 @@ class GatewayShutdownMixin: if retrying_since is not _UNSET: extra["retrying_since"] = retrying_since write_runtime_status( - platform=platform, - platform_state=platform_state, - error_code=error_code, - error_message=error_message, - **extra, + platform=platform, platform_state=platform_state, error_code=error_code, + error_message=error_message, **extra, ) - except Exception: - pass - # ------------------------------------------------------------------ # Per-platform circuit breaker (pause/resume): reconnect watcher + /platform pause|resume. - # ------------------------------------------------------------------ def _pause_failed_platform(self, platform, *, reason: str = "") -> None: - """Mark a queued platform as paused — stays in ``_failed_platforms`` but the reconnect - watcher stops hammering it. - - Manual (``/platform pause ``) only: the watcher never auto-pauses — retryable failures - keep retrying at the backoff cap so a transient outage self-heals. - """ + """Pause a queued platform (manual ``/platform pause`` only — the watcher never auto-pauses).""" info = getattr(self, "_failed_platforms", {}).get(platform) - if info is None: - return - if info.get("paused"): + if info is None or info.get("paused"): return info["paused"] = True info["pause_reason"] = reason or "auto-paused after repeated failures" - # Push next_retry far enough out that even if "paused" is missed - # by a stale code path, the watcher won't fire on it. + # Push next_retry far enough out that a stale code path missing "paused" still never fires. info["next_retry"] = float("inf") with suppress(Exception): self._update_platform_runtime_status( - platform.value, - platform_state="paused", - error_code=None, - error_message=info["pause_reason"], + platform.value, platform_state="paused", error_code=None, error_message=info["pause_reason"], ) logger.warning( "%s paused after %d consecutive failures (%s) — " "fix the underlying issue then run `/platform resume %s` " "to retry, or `hermes gateway restart` to restart the gateway.", - platform.value, info.get("attempts", 0), - info["pause_reason"], platform.value, + platform.value, info.get("attempts", 0), info["pause_reason"], platform.value, ) def _resume_paused_platform(self, platform) -> bool: - """Unpause a platform — reset its attempt counter and schedule an - immediate retry. Returns True if the platform was paused and is - now queued; False if it wasn't paused (or wasn't in the queue). - """ + """Unpause a platform (reset attempts, retry on the next watcher tick). True iff it was paused.""" info = getattr(self, "_failed_platforms", {}).get(platform) - if info is None: - return False - if not info.get("paused"): + if info is None or not info.get("paused"): return False info["paused"] = False info.pop("pause_reason", None) @@ -622,90 +546,58 @@ class GatewayShutdownMixin: info["next_retry"] = time.monotonic() # retry on next watcher tick with suppress(Exception): self._update_platform_runtime_status( - platform.value, - platform_state="retrying", - error_code=None, - error_message=None, + platform.value, platform_state="retrying", error_code=None, error_message=None, ) logger.info("%s resumed — retrying on next watcher tick", platform.value) return True + # Drain / interrupt + def _drain_work_counts(self) -> tuple: + """``(agents, cron, api, deferred)`` — the four sources the drain waits on.""" + return ( + self._running_agent_count(), self._active_cron_job_count(), + self._active_api_run_count(), self._active_deferred_agent_worker_count(), + ) + async def _drain_active_agents( self, timeout: float, cron_timeout: Optional[float] = None ) -> tuple[Dict[str, Any], bool]: snapshot = self._snapshot_running_agents() - last_active_count = self._running_agent_count() - last_cron_count = self._active_cron_job_count() - last_api_count = self._active_api_run_count() - last_deferred_count = self._active_deferred_agent_worker_count() + loop = asyncio.get_running_loop() + last_counts = self._drain_work_counts() last_status_at = 0.0 def _maybe_update_status(force: bool = False) -> None: - nonlocal last_active_count, last_cron_count, last_api_count - nonlocal last_deferred_count, last_status_at - now = asyncio.get_running_loop().time() - active_count = self._running_agent_count() - cron_count = self._active_cron_job_count() - api_count = self._active_api_run_count() - deferred_count = self._active_deferred_agent_worker_count() - if ( - force - or active_count != last_active_count - or cron_count != last_cron_count - or api_count != last_api_count - or deferred_count != last_deferred_count - or (now - last_status_at) >= 1.0 - ): + nonlocal last_counts, last_status_at + now = loop.time() + counts = self._drain_work_counts() + if force or counts != last_counts or (now - last_status_at) >= 1.0: self._update_runtime_status("draining") - last_active_count = active_count - last_cron_count = cron_count - last_api_count = api_count - last_deferred_count = deferred_count - last_status_at = now - - # Cron jobs run on the scheduler's pool, outside ``self._running_agents`` — fold their in-flight - # count into this wait, or a cron job's tool work is killed without warning once it's the only - # active thing running. API-server/desk sessions and detached deferred workers share the gap. - if ( - not self._running_agents - and last_cron_count == 0 - and last_api_count == 0 - and last_deferred_count == 0 - ): - _maybe_update_status(force=True) - return snapshot, False + last_counts, last_status_at = counts, now + # Cron/API-server/deferred work lives outside ``self._running_agents`` — fold it into this + # wait, or a cron job's tool work is killed without warning once it's the only thing running. + _cron0, _api0, _deferred0 = last_counts[1:] _maybe_update_status(force=True) - - # Cron drains on its own deadline: ``timeout`` (``restart_drain_timeout``) defaults to 0 since - # an interrupted chat turn is announced and resumable, while a cron run killed mid-flight is a - # permanent failure nobody is waiting on. One shared budget would kill cron after 0.00s. - loop = asyncio.get_running_loop() + if not self._running_agents and not (_cron0 or _api0 or _deferred0): + return snapshot, False + # Cron has its own deadline: ``restart_drain_timeout`` defaults to 0 (an interrupted chat turn + # is announced and resumable) but a cron run killed mid-flight is a permanent failure. started = loop.time() deadline = started + timeout cron_deadline = started + (timeout if cron_timeout is None else cron_timeout) def _still_draining() -> bool: now = loop.time() - if ( - len(self._running_agents) - or self._active_api_run_count() - or self._active_deferred_agent_worker_count() - ) and now < deadline: - return True - return bool(self._active_cron_job_count()) and now < cron_deadline + agents, cron, api, deferred = self._drain_work_counts() + return bool(((agents or api or deferred) and now < deadline) or (cron and now < cron_deadline)) # Both budgets at 0 leave this loop unentered ("interrupt immediately") as an expired deadline, # not a special case, so timed_out below is always computed from real state. while _still_draining(): _maybe_update_status() await asyncio.sleep(0.1) - timed_out = ( - bool(len(self._running_agents)) - or bool(self._active_cron_job_count()) - or bool(self._active_api_run_count()) - or bool(self._active_deferred_agent_worker_count()) - ) + timed_out = any(self._drain_work_counts()) _maybe_update_status(force=True) return snapshot, timed_out @@ -726,18 +618,37 @@ class GatewayShutdownMixin: logger.debug("Interrupted %d api_server run(s) during shutdown", interrupted_api) interrupted_deferred = self._interrupt_deferred_agent_workers(reason) if interrupted_deferred: - logger.debug( - "Interrupted %d deferred agent worker(s) during shutdown", - interrupted_deferred, - ) + logger.debug("Interrupted %d deferred agent worker(s) during shutdown", interrupted_deferred) + + def _shutdown_interrupt_reason(self) -> str: + from gateway.run import _INTERRUPT_REASON_GATEWAY_RESTART, _INTERRUPT_REASON_GATEWAY_SHUTDOWN + return _INTERRUPT_REASON_GATEWAY_RESTART if self._restart_requested else _INTERRUPT_REASON_GATEWAY_SHUTDOWN + + async def _mark_running_sessions_resume_pending(self, log_prefix: str) -> list: + """Mark every non-pending running session resume_pending; returns the keys marked.""" + from gateway.run import _AGENT_PENDING_SENTINEL + reason = "restart_timeout" if self._restart_requested else "shutdown_timeout" + marked: list[str] = [] + for _sk, _agent in list(self._running_agents.items()): + if _agent is _AGENT_PENDING_SENTINEL: + continue + try: + await self.async_session_store.mark_resume_pending(_sk, reason) + marked.append(_sk) + except Exception as _e: + logger.debug("%s failed for %s: %s", log_prefix, _sk, _e) + return marked + + def _restart_notification_allowed(self, platform: Platform) -> bool: + """False when the platform config sets ``gateway_restart_notification=false``.""" + platform_cfg = self.config.platforms.get(platform) + return platform_cfg is None or bool(platform_cfg.gateway_restart_notification) async def _notify_interrupted_cron_jobs(self, job_ids) -> int: - """Tell the owner of each just-interrupted cron job that its run died. + """Tell the owner of each just-interrupted cron job that its run died; returns notices sent. - The cron worker can't: its thread reaches ``_deliver_result`` after teardown closed the - transport. Must run post-interrupt while adapters are still connected (the window - ``_notify_active_sessions_of_shutdown`` uses, which is blind to cron work). Best-effort: every - failure is swallowed so a wedged adapter can't extend shutdown. Returns notices sent. + The cron worker can't (its thread reaches ``_deliver_result`` after teardown closed the + transport), so this runs post-interrupt while adapters are still connected. Best-effort. """ if not job_ids: return 0 @@ -747,7 +658,6 @@ class GatewayShutdownMixin: except Exception as e: logger.debug("Cron interrupt notification unavailable: %s", e) return 0 - action = "restarting" if self._restart_requested else "shutting down" notified: set = set() for job_id in job_ids: @@ -762,70 +672,85 @@ class GatewayShutdownMixin: except Exception as e: logger.debug("Cron interrupt targets unresolved for %s: %s", job_id, e) continue - if not targets: - continue - msg = ( f"⚠️ Cron job '{job.get('name') or job_id}' was interrupted — " f"the gateway is {action} and killed the run before it " "finished. No result was produced for this run." ) - for target in targets: + for target in targets or (): try: platform = Platform(str(target.get("platform", "")).lower()) except Exception: continue adapter = self.adapters.get(platform) - if adapter is None: + if adapter is None or not self._restart_notification_allowed(platform): continue - platform_cfg = self.config.platforms.get(platform) - if platform_cfg is not None and not platform_cfg.gateway_restart_notification: - continue - chat_id = str(target.get("chat_id")) thread_id = target.get("thread_id") - dedup_key = ( - job_id, - platform.value, - chat_id, - str(thread_id) if thread_id else None, - ) + dedup_key = (job_id, platform.value, chat_id, str(thread_id) if thread_id else None) if dedup_key in notified: continue try: - metadata = self._thread_metadata_for_target( - platform, chat_id, thread_id, adapter=adapter - ) + metadata = self._thread_metadata_for_target(platform, chat_id, thread_id, adapter=adapter) result = await adapter.send(chat_id, msg, metadata=metadata) - if result is not None and getattr(result, "success", True) is False: + if _send_failed(result): logger.debug( - "Cron interrupt notice to %s:%s failed: %s", - platform.value, chat_id, + "Cron interrupt notice to %s:%s failed: %s", platform.value, chat_id, getattr(result, "error", "send returned success=False"), ) continue notified.add(dedup_key) except Exception as e: - logger.debug( - "Cron interrupt notice to %s:%s raised: %s", - platform.value, chat_id, e, - ) + logger.debug("Cron interrupt notice to %s:%s raised: %s", platform.value, chat_id, e) if notified: - logger.info( - "Shutdown: delivered %d interrupted-cron-job notice(s)", - len(notified), - ) + logger.info("Shutdown: delivered %d interrupted-cron-job notice(s)", len(notified)) return len(notified) + async def _shutdown_notification_target(self, session_key: str): + """``(source, platform_str, chat_id, thread_id)``: persisted origin > cached source > parsed key.""" + from gateway.run import _parse_session_key + source = None + try: + if getattr(self, "session_store", None) is not None: + await self.async_session_store._ensure_loaded() + entry = self.session_store._entries.get(session_key) + source = getattr(entry, "origin", None) if entry else None + except Exception as e: + logger.debug("Failed to load session origin for shutdown notification %s: %s", session_key, e) + if source is None: + source = self._get_cached_session_source(session_key) + if source is not None: + return source, source.platform.value, str(source.chat_id), source.thread_id + _parsed = _parse_session_key(session_key) + if not _parsed: + return None + return None, _parsed["platform"], _parsed["chat_id"], _parsed.get("thread_id") + + async def _send_shutdown_notice( + self, adapter, chat_id: str, msg: str, kind: str, platform_str: str, **send_kwargs + ) -> bool: + """Send one shutdown notice; True when delivered. Failures are debug-logged, never raised.""" + where = "home channel " if kind == "home channel" else "" + try: + result = await adapter.send(chat_id, msg, **send_kwargs) + if _send_failed(result): + logger.debug( + "Failed to send shutdown notification to %s%s:%s: %s", + where, platform_str, chat_id, getattr(result, "error", "send returned success=False"), + ) + return False + logger.info("Sent shutdown notification to %s %s:%s", kind, platform_str, chat_id) + return True + except Exception as e: + logger.debug("Failed to send shutdown notification to %s%s:%s: %s", where, platform_str, chat_id, e) + return False + async def _notify_active_sessions_of_shutdown(self) -> None: """Send shutdown/restart notifications to active chats and home channels. Called at the start of stop() while adapters are connected; send failures never block shutdown. """ - from gateway.run import _parse_session_key - active = self._snapshot_running_agents() restart_source = self._restart_command_source if self._restart_requested else None - action = "restarting" if self._restart_requested else "shutting down" hint = ( "Your current task will be interrupted. " @@ -834,239 +759,144 @@ class GatewayShutdownMixin: else "Your current task will be interrupted." ) msg = f"⚠️ Gateway {action} — {hint}" - - notified: set[tuple[str, str, Optional[str]]] = set() - for session_key in active: - source = None - try: - if getattr(self, "session_store", None) is not None: - await self.async_session_store._ensure_loaded() - entry = self.session_store._entries.get(session_key) - source = getattr(entry, "origin", None) if entry else None - except Exception as e: - logger.debug( - "Failed to load session origin for shutdown notification %s: %s", - session_key, - e, + restart_key = None + if restart_source is not None: + with suppress(Exception): + restart_key = ( + restart_source.platform.value, str(restart_source.chat_id), + str(restart_source.thread_id) if restart_source.thread_id else None, ) - - if source is None: - source = self._get_cached_session_source(session_key) - - if source is not None: - platform_str = source.platform.value - chat_id = str(source.chat_id) - thread_id = source.thread_id - else: - # Fall back to parsing the session key when no persisted - # origin is available (legacy sessions/tests). - _parsed = _parse_session_key(session_key) - if not _parsed: - continue - platform_str = _parsed["platform"] - chat_id = _parsed["chat_id"] - thread_id = _parsed.get("thread_id") - + notified: set[tuple[str, str, Optional[str]]] = set() + for session_key in self._snapshot_running_agents(): + target = await self._shutdown_notification_target(session_key) + if target is None: + continue + source, platform_str, chat_id, thread_id = target # Dedupe only identical targets: thread/topic platforms share a parent chat yet route to # distinct destinations via metadata. dedup_key = (platform_str, chat_id, str(thread_id) if thread_id else None) if dedup_key in notified: continue - try: platform = Platform(platform_str) adapter = self.adapters.get(platform) if not adapter: continue - - platform_cfg = self.config.platforms.get(platform) - if platform_cfg is not None and not platform_cfg.gateway_restart_notification: + if not self._restart_notification_allowed(platform): logger.info( "Shutdown notification suppressed for active session: %s has gateway_restart_notification=false", platform_str, ) continue - - reply_to_message_id = getattr(source, "message_id", None) if source is not None else None - if reply_to_message_id is None and restart_source is not None: - try: - restart_platform = restart_source.platform.value - restart_chat_id = str(restart_source.chat_id) - restart_thread_id = str(restart_source.thread_id) if restart_source.thread_id else None - if (restart_platform, restart_chat_id, restart_thread_id) == dedup_key: - reply_to_message_id = getattr(restart_source, "message_id", None) - except Exception: - pass - + reply_to_message_id = getattr(source, "message_id", None) + if reply_to_message_id is None and restart_key == dedup_key: + reply_to_message_id = getattr(restart_source, "message_id", None) metadata = self._thread_metadata_for_target( - platform, - chat_id, - thread_id, - chat_type=getattr(source, "chat_type", None) if source is not None else None, - reply_to_message_id=reply_to_message_id, - adapter=adapter, - ) - - result = await adapter.send(chat_id, msg, metadata=metadata) - if result is not None and getattr(result, "success", True) is False: - logger.debug( - "Failed to send shutdown notification to %s:%s: %s", - platform_str, - chat_id, - getattr(result, "error", "send returned success=False"), - ) - continue - - notified.add(dedup_key) - logger.info( - "Sent shutdown notification to active chat %s:%s", - platform_str, chat_id, + platform, chat_id, thread_id, chat_type=getattr(source, "chat_type", None), + reply_to_message_id=reply_to_message_id, adapter=adapter, ) except Exception as e: - logger.debug( - "Failed to send shutdown notification to %s:%s: %s", - platform_str, chat_id, e, - ) - + logger.debug("Failed to send shutdown notification to %s:%s: %s", platform_str, chat_id, e) + continue + if await self._send_shutdown_notice( + adapter, chat_id, msg, "active chat", platform_str, metadata=metadata + ): + notified.add(dedup_key) if self._restart_requested and restart_source is not None: logger.debug("Skipping home-channel shutdown notifications for in-chat restart") return - - # Suppress ONLY the home-channel broadcast when the drain asked to be quiet (e.g. routine - # auto-update on an always-on fleet). Per-session interrupt pings above are NOT gated: empty by - # construction on a drained shutdown, and useful ("task cut off, message me to resume") on a - # force-interrupt. Honoured only for a CURRENT-epoch marker (staleness check inside - # drain_notification_suppressed), so an orphaned marker can't silence a fresh gateway. + # Suppress ONLY the home-channel broadcast when the drain asked to be quiet (routine fleet + # auto-update); per-session pings above stay (empty on a drained shutdown, useful on a + # force-interrupt). Current-epoch marker only, so an orphaned marker can't silence a fresh gateway. try: from gateway.drain_control import drain_notification_suppressed if drain_notification_suppressed(): logger.info( - "Home-channel shutdown broadcast suppressed by drain marker " - "(suppress_notification=true)" + "Home-channel shutdown broadcast suppressed by drain marker (suppress_notification=true)" ) return except Exception as e: - # Never let the suppression check block the shutdown broadcast — - # fail toward the louder, more-visible behaviour. + # Never let the suppression check block the broadcast — fail toward the louder behaviour. logger.debug("drain_notification_suppressed check failed: %s", e) - # Snapshot adapters: adapter.send() can hit a fatal path (_handle_fatal) that pops the adapter # from self.adapters -> ``RuntimeError: dictionary changed size during iteration``. for platform, adapter in list(self.adapters.items()): home = self.config.get_home_channel(platform) if not home or not home.chat_id: continue - - platform_cfg = self.config.platforms.get(platform) - if platform_cfg is not None and not platform_cfg.gateway_restart_notification: + if not self._restart_notification_allowed(platform): logger.info( "Shutdown notification suppressed for home channel: %s has gateway_restart_notification=false", platform.value, ) continue - dedup_key = (platform.value, str(home.chat_id), str(home.thread_id) if home.thread_id else None) if dedup_key in notified: continue - try: - metadata = self._thread_metadata_for_target( - platform, - home.chat_id, - home.thread_id, - adapter=adapter, - ) - if metadata: - result = await adapter.send(str(home.chat_id), msg, metadata=metadata) - else: - result = await adapter.send(str(home.chat_id), msg) - if result is not None and getattr(result, "success", True) is False: - logger.debug( - "Failed to send shutdown notification to home channel %s:%s: %s", - platform.value, - home.chat_id, - getattr(result, "error", "send returned success=False"), - ) - continue - - notified.add(dedup_key) - logger.info( - "Sent shutdown notification to home channel %s:%s", - platform.value, - home.chat_id, - ) + metadata = self._thread_metadata_for_target(platform, home.chat_id, home.thread_id, adapter=adapter) except Exception as e: logger.debug( - "Failed to send shutdown notification to home channel %s:%s: %s", - platform.value, - home.chat_id, - e, + "Failed to send shutdown notification to home channel %s:%s: %s", platform.value, home.chat_id, e, ) + continue + # Home channels omit ``metadata=`` when empty (adapter doubles may not accept the kwarg). + send_kwargs = {"metadata": metadata} if metadata else {} + if await self._send_shutdown_notice( + adapter, str(home.chat_id), msg, "home channel", platform.value, **send_kwargs + ): + notified.add(dedup_key) + + # Agent finalization / resource cleanup + @staticmethod + def _flush_agent_transcript_at_shutdown(agent: Any) -> None: + """Persist an in-flight transcript before teardown. + + A force-interrupted agent may never reach finalize_turn (the only mid-turn flush), so its + tool rounds would vanish on resume. Idempotent; gracefully finished agents re-flush nothing. + """ + try: + _flush = getattr(agent, "_flush_messages_to_session_db", None) + _session_messages = getattr(agent, "_session_messages", None) + if not (callable(_flush) and isinstance(_session_messages, list) and _session_messages): + return + # Strip empty-response retry scaffolding from the tail first (as ``_persist_session`` + # does) so a resumed turn doesn't replay synthetic recovery nudges. + _strip = getattr(agent, "_drop_trailing_empty_response_scaffolding", None) + if callable(_strip): + with suppress(Exception): + _strip(_session_messages) + try: + _flush(_session_messages) + except Exception as _flush_err: + # Transcript could not be persisted (e.g. FTS/SQLite corruption): dump the live history + # to a JSON recovery snapshot rather than lose it. Non-fatal. + logger.warning( + "Shutdown transcript flush failed (%s); preserving %d in-memory message(s) to recovery snapshot", + _flush_err, len(_session_messages), + ) + from gateway.shutdown_flush import flush_agent_history_to_file + flush_agent_history_to_file(getattr(agent, "session_id", None), _session_messages) + except Exception as _e: + logger.debug("Shutdown transcript flush failed: %s", _e) async def _finalize_shutdown_agents(self, active_agents: Dict[str, Any]) -> None: for agent in active_agents.values(): - # Persist in-flight transcripts before teardown: a force-interrupted agent may never reach - # finalize_turn (the only mid-turn flush), so its tool rounds would vanish from - # load_transcript() on resume (resume already tolerates a pending-tool-result tail). The - # flush is idempotent (identity-tracked); gracefully finished agents re-flush nothing. - try: - _flush = getattr(agent, "_flush_messages_to_session_db", None) - _session_messages = getattr(agent, "_session_messages", None) - if callable(_flush) and isinstance(_session_messages, list) and _session_messages: - # Strip empty-response retry scaffolding from the tail first (as ``_persist_session`` - # does) so a resumed turn doesn't replay synthetic recovery nudges. - _strip = getattr( - agent, "_drop_trailing_empty_response_scaffolding", None - ) - if callable(_strip): - with suppress(Exception): - _strip(_session_messages) - try: - _flush(_session_messages) - except Exception as _flush_err: - # Transcript could not be persisted (e.g. FTS/SQLite index corruption). A log - # line alone loses the conversation at exit, so dump the live history to an - # external JSON recovery snapshot. Non-fatal: shutdown never blocks on a backup. - logger.warning( - "Shutdown transcript flush failed (%s); preserving " - "%d in-memory message(s) to recovery snapshot", - _flush_err, - len(_session_messages), - ) - from gateway.shutdown_flush import flush_agent_history_to_file - flush_agent_history_to_file( - getattr(agent, "session_id", None), - _session_messages, - ) - except Exception as _e: - logger.debug("Shutdown transcript flush failed: %s", _e) + self._flush_agent_transcript_at_shutdown(agent) # Off-loop + bounded: plugin on_session_finalize hooks can do arbitrary synchronous work # (e.g. a full-session trace export) — same hang class as the memory provider below. await self._finalize_session_off_loop( - session_id=getattr(agent, "session_id", None), - platform="gateway", - reason="shutdown", - ) - # Off-loop + bounded: a wedged memory provider here used to hang - # the whole shutdown so SIGTERM never completed (#53175). - await self._cleanup_agent_resources_off_loop( - agent, context="shutdown finalize" + session_id=getattr(agent, "session_id", None), platform="gateway", reason="shutdown", ) + # Off-loop + bounded: a wedged memory provider here used to hang the whole shutdown so + # SIGTERM never completed. + await self._cleanup_agent_resources_off_loop(agent, context="shutdown finalize") def _should_emit_long_running_notification( - self, - session_key: Optional[str], - agent: Any, - executor_task: Optional[Any], + self, session_key: Optional[str], agent: Any, executor_task: Optional[Any], ) -> bool: - """Only emit the heartbeat while this task still owns the live run. - - Stop once the executor finishes, the agent is gone, or the session key was rebound (e.g. - ``/new`` mid-run) — else a stale ``running: delegate_task`` heartbeat outlives its run. - """ - if agent is None: - return False - if executor_task is not None and executor_task.done(): + """Emit the heartbeat only while this task still owns the live run (not after ``/new`` rebinds).""" + if agent is None or (executor_task is not None and executor_task.done()): return False if session_key: _hb_state = self._peek_session_state(session_key) @@ -1074,133 +904,80 @@ class GatewayShutdownMixin: return False return True - def _defer_agent_cleanup_until_future_done( - self, - future: asyncio.Future, - agent: Any, - *, - context: str, - ) -> None: - """Clean up ``agent`` only after its executor future has finished. - - A timed-out executor call keeps running in its worker thread; closing the agent first can - tear down clients it still uses, so hold a strong task ref and await the real future. - """ + def _defer_agent_cleanup_until_future_done(self, future: asyncio.Future, agent: Any, *, context: str) -> None: + """Clean up ``agent`` only after its executor future finishes (it may still use the agent's clients).""" async def _cleanup_when_done() -> None: try: await asyncio.shield(future) except asyncio.CancelledError: - # Loop shutdown can cancel this waiter while the executor still - # runs. Never turn that cancellation into premature cleanup. + # Loop shutdown can cancel this waiter while the executor still runs. Never turn + # that cancellation into premature cleanup. return except Exception as exc: logger.debug( - "Deferred agent worker%s finished with an error: %s", - f" ({context})" if context else "", - exc, + "Deferred agent worker%s finished with an error: %s", f" ({context})" if context else "", exc, ) await self._cleanup_agent_resources_off_loop(agent, context=context) self._track_deferred_agent_worker(future, agent) - task = asyncio.create_task(_cleanup_when_done()) tasks = getattr(self, "_deferred_agent_cleanup_tasks", None) if tasks is None: - tasks = set() - self._deferred_agent_cleanup_tasks = tasks + tasks = self._deferred_agent_cleanup_tasks = set() tasks.add(task) task.add_done_callback(tasks.discard) - async def _finalize_session_off_loop( - self, - *, - session_id: Any, - platform: str, - reason: str, - **extra: Any, - ) -> None: - """Run hermes_cli.lifecycle.finalize_session off the event loop, bounded. - - On timeout the worker thread is left to finish (or leak) and the caller proceeds. - """ + async def _finalize_session_off_loop(self, *, session_id: Any, platform: str, reason: str, **extra: Any) -> None: + """Run hermes_cli.lifecycle.finalize_session off-loop, bounded; on timeout the worker is left alone.""" def _call() -> None: from hermes_cli.lifecycle import finalize_session - - finalize_session( - session_id=session_id, - platform=platform, - reason=reason, - **extra, - ) + finalize_session(session_id=session_id, platform=platform, reason=reason, **extra) try: - await asyncio.wait_for( - self._run_in_executor_with_context(_call), - timeout=self._FINALIZE_TIMEOUT_S, - ) + await asyncio.wait_for(self._run_in_executor_with_context(_call), timeout=self._FINALIZE_TIMEOUT_S) except asyncio.TimeoutError: logger.warning( "Session finalize hooks (%s, reason=%s) exceeded %ss; " "proceeding without blocking the event loop (the worker " "thread is left to finish on its own).", - session_id, - reason, - self._FINALIZE_TIMEOUT_S, + session_id, reason, self._FINALIZE_TIMEOUT_S, ) except Exception as finalize_exc: - logger.debug( - "Session finalize hooks (%s, reason=%s) failed: %s", - session_id, - reason, - finalize_exc, - ) + logger.debug("Session finalize hooks (%s, reason=%s) failed: %s", session_id, reason, finalize_exc) - async def _cleanup_agent_resources_off_loop( - self, agent: Any, *, context: str = "" - ) -> None: - """Run _cleanup_agent_resources in a worker thread with a bounded wait. - - On timeout the worker thread is left to finish (or leak) and the caller proceeds, as /new does. - """ + async def _cleanup_agent_resources_off_loop(self, agent: Any, *, context: str = "") -> None: + """Run _cleanup_agent_resources in a worker thread, bounded; on timeout the worker is left alone.""" if agent is None: return if context.startswith("shutdown") or context == "session expiry": with suppress(Exception): agent._end_session_on_close = False + ctx_label = f" ({context})" if context else "" try: await asyncio.wait_for( - self._run_in_executor_with_context( - self._cleanup_agent_resources, agent - ), + self._run_in_executor_with_context(self._cleanup_agent_resources, agent), timeout=self._CLEANUP_TIMEOUT_S, ) except asyncio.TimeoutError: logger.warning( "Agent resource cleanup%s exceeded %ss; proceeding without " "blocking the event loop (the worker thread is left to finish " - "on its own). (#53175)", - f" ({context})" if context else "", - self._CLEANUP_TIMEOUT_S, + "on its own). (#53175)", ctx_label, self._CLEANUP_TIMEOUT_S, ) except Exception as cleanup_exc: - logger.warning( - "Agent resource cleanup%s failed: %s (#53175)", - f" ({context})" if context else "", - cleanup_exc, - ) + logger.warning("Agent resource cleanup%s failed: %s (#53175)", ctx_label, cleanup_exc) def _cleanup_agent_resources(self, agent: Any) -> None: """Best-effort cleanup for temporary or cached agent instances.""" if agent is None: return - try: + with suppress(Exception): if hasattr(agent, "shutdown_memory_provider"): - # Drain queued memory writes BEFORE tearing the provider down: shutdown_all() gives - # the serialized memory worker only ~5s and cancels the rest, so a /reset or rotation - # could drop handed-off writes and the next session loads stale memory. Bounded head - # start via the manager's own barrier (mirrors CLI exit); a failure never blocks teardown. + # Drain queued memory writes BEFORE teardown: shutdown_all() gives the memory worker + # only ~5s, so a /reset or rotation could drop handed-off writes (stale memory next + # session). Bounded, mirrors CLI exit; a failure never blocks teardown. _mm = getattr(agent, "_memory_manager", None) if _mm is not None and hasattr(_mm, "flush_pending"): with suppress(Exception): @@ -1212,115 +989,145 @@ class GatewayShutdownMixin: agent.shutdown_memory_provider(session_messages) else: agent.shutdown_memory_provider() - except Exception: - pass # Close tool resources (sandboxes, browser daemons, background processes, httpx clients). - try: + with suppress(Exception): if hasattr(agent, "close"): agent.close() - except Exception: - pass # Auxiliary async clients live in a process-global cache created from worker threads; drop # entries whose event loop is dead so httpx transports don't accumulate across turns. - try: + with suppress(Exception): from agent.auxiliary_client import cleanup_stale_async_clients cleanup_stale_async_clients() + + # Stuck-loop (restart failure) counters + def _stuck_loop_counts_path(self) -> Path: + from gateway.run import _hermes_home + return _hermes_home / self._STUCK_LOOP_FILE + + @staticmethod + def _read_json_counts(path: Path) -> Optional[dict]: + """Parsed counter dict, or None when the file is missing/unreadable.""" + try: + return json.loads(path.read_text(encoding="utf-8")) except Exception: - pass + return None def _increment_restart_failure_counts(self, active_session_keys: set) -> None: - """Increment restart-failure counters for sessions active at shutdown. - - Persists to a JSON file so counters survive across restarts. Sessions NOT in - active_session_keys are removed (they completed successfully, so the loop is broken). - """ - from gateway.run import _hermes_home, atomic_json_write - import json - - path = _hermes_home / self._STUCK_LOOP_FILE - try: - counts = json.loads(path.read_text(encoding="utf-8")) if path.exists() else {} - except Exception: - counts = {} - - # Increment active sessions, remove inactive ones (loop broken) - new_counts = {} - for key in active_session_keys: - new_counts[key] = counts.get(key, 0) + 1 - # Keep any entries that are still above 0 even if not active now - # (they might become active again next restart) - + """Increment persisted restart-failure counters for active sessions; drop the rest (loop broken).""" + from gateway.run import atomic_json_write + path = self._stuck_loop_counts_path() + counts = (self._read_json_counts(path) if path.exists() else None) or {} + new_counts = {key: counts.get(key, 0) + 1 for key in active_session_keys} with suppress(Exception): atomic_json_write(path, new_counts, indent=None) def _suspend_stuck_loop_sessions(self) -> int: - """Suspend sessions active across too many restarts (load → stuck → restart loop). - - Runs at startup AFTER suspend_recently_active(). Returns the number suspended. - """ - from gateway.run import _hermes_home - import json - - path = _hermes_home / self._STUCK_LOOP_FILE + """Suspend sessions active across too many restarts (startup, AFTER suspend_recently_active()).""" + path = self._stuck_loop_counts_path() if not path.exists(): return 0 - - try: - counts = json.loads(path.read_text(encoding="utf-8")) - except Exception: + counts = self._read_json_counts(path) + if counts is None: return 0 - suspended = 0 - stuck_keys = [k for k, v in counts.items() if v >= self._STUCK_LOOP_THRESHOLD] - - for session_key in stuck_keys: + for session_key in [k for k, v in counts.items() if v >= self._STUCK_LOOP_THRESHOLD]: try: entry = self.session_store._entries.get(session_key) if entry and not entry.suspended: entry.suspended = True suspended += 1 logger.warning( - "Auto-suspended stuck session %s (active across %d " - "consecutive restarts — likely a stuck loop)", + "Auto-suspended stuck session %s (active across %d consecutive restarts — likely a stuck loop)", session_key, counts[session_key], ) except Exception: pass - if suspended: with suppress(Exception): self.session_store._save() - # Clear the file — counters start fresh after suspension with suppress(Exception): path.unlink(missing_ok=True) - return suspended async def _clear_restart_failure_count(self, session_key: str) -> None: """Clear a completed session's restart-failure counter off-loop (atomic_json_write fsyncs).""" - from gateway.run import _hermes_home, atomic_json_write - import json - - path = _hermes_home / self._STUCK_LOOP_FILE + from gateway.run import atomic_json_write + path = self._stuck_loop_counts_path() if not path.exists(): return + counts = self._read_json_counts(path) or {} + if session_key not in counts: + return + del counts[session_key] + with suppress(Exception): + if counts: + await asyncio.to_thread(atomic_json_write, path, counts, indent=None) + else: + path.unlink(missing_ok=True) + + # Restart orchestration + @staticmethod + def _restart_watcher_env() -> dict: + """Watcher env minus ``_HERMES_GATEWAY`` (else the CLI's self-restart guard refuses; gateway stays down).""" + from tools.environments.local import build_subprocess_env + watcher_env = build_subprocess_env(scrub_secrets=False, inherit_profile_home=True) + watcher_env.pop("_HERMES_GATEWAY", None) + return watcher_env + + @staticmethod + def _spawn_windows_restart_watcher(hermes_cmd: list, current_pid: int, restart_after_s: float) -> None: + """Spawn the detached Windows watcher (``python -c``), retrying once without job breakaway.""" + import subprocess + from hermes_cli._subprocess_compat import ( + windows_detach_flags_without_breakaway, windows_detach_popen_kwargs + ) + watcher_env = GatewayShutdownMixin._restart_watcher_env() + project_root = Path(__file__).resolve().parent.parent + # Console python under CREATE_NO_WINDOW owns one hidden console inherited by the restart + # child, so nothing flashes. Do NOT swap in pythonw.exe — a console-less watcher forces + # every console-subsystem descendant to allocate a visible conhost. + watcher_python = sys.executable + venv_dir = Path(watcher_env.get("VIRTUAL_ENV") or project_root / "venv") + site_packages = venv_dir / "Lib" / "site-packages" + if site_packages.exists(): + watcher_env["VIRTUAL_ENV"] = str(venv_dir) + pythonpath = [str(project_root), str(site_packages)] + if watcher_env.get("PYTHONPATH"): + pythonpath.append(watcher_env["PYTHONPATH"]) + watcher_env["PYTHONPATH"] = os.pathsep.join(dict.fromkeys(pythonpath)) + watcher_argv = [ + watcher_python, "-c", _WINDOWS_RESTART_WATCHER, + str(current_pid), str(restart_after_s), *hermes_cmd, "gateway", "restart", + ] + popen_kwargs = dict(stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, env=watcher_env) + # The watcher must break away from the parent CLI's job object (Desktop wrappers, Windows + # Terminal, schtasks) or it is reaped when the CLI exits. A job without + # JOB_OBJECT_LIMIT_BREAKAWAY_OK rejects CREATE_BREAKAWAY_FROM_JOB (OSError); retry once + # without the bit, preserving argv and the scrubbed env. try: - counts = json.loads(path.read_text(encoding="utf-8")) - if session_key in counts: - del counts[session_key] - if counts: - await asyncio.to_thread(atomic_json_write, path, counts, indent=None) - else: - path.unlink(missing_ok=True) - except Exception: - pass + subprocess.Popen(watcher_argv, **popen_kwargs, **windows_detach_popen_kwargs()) + except OSError: + try: + subprocess.Popen( + watcher_argv, **popen_kwargs, creationflags=windows_detach_flags_without_breakaway(), + ) + except OSError as exc: + # Both spawns failed. Log only the interpreter basename and numeric errno — never + # argv, env, watcher source, or str(exc) (may carry a full path) — and return. + winerror = getattr(exc, "winerror", None) + logger.warning( + "Detached restart watcher was not started after the " + "no-breakaway retry (%s; %s=%r). The gateway will not " + "be respawned by this restart attempt.", os.path.basename(watcher_python), + "winerror" if winerror is not None else "errno", + winerror if winerror is not None else exc.errno, + ) async def _launch_detached_restart_command(self) -> None: from gateway.run import _resolve_hermes_bin import shutil import subprocess - hermes_cmd = _resolve_hermes_bin() if not hermes_cmd: logger.error("Could not locate hermes binary for detached /restart") @@ -1328,238 +1135,86 @@ class GatewayShutdownMixin: if self._detached_restart_helper_started: return self._detached_restart_helper_started = True - current_pid = os.getpid() restart_after_s = max(float(getattr(self, "_restart_drain_timeout", 0.0) or 0.0) + 5.0, 5.0) - - # On Windows there's no bash/setsid chain — spawn a tiny Python watcher directly via - # sys.executable instead. if sys.platform == "win32": - import textwrap - from hermes_cli._subprocess_compat import ( - windows_detach_flags_without_breakaway, - windows_detach_popen_kwargs, - ) - - cmd_argv = [*hermes_cmd, "gateway", "restart"] - watcher = textwrap.dedent( - """ - import os, subprocess, sys, time - from hermes_cli._subprocess_compat import windows_detach_flags_without_breakaway - pid = int(sys.argv[1]) - restart_after_s = float(sys.argv[2]) - cmd = sys.argv[3:] - deadline = time.monotonic() + restart_after_s - - def _alive(p): - # On Windows, os.kill(pid, 0) is NOT a no-op — it maps to - # GenerateConsoleCtrlEvent(0, pid) (bpo-14484). Use the - # Win32 handle-based existence check instead. - if os.name == 'nt': - import ctypes - k32 = ctypes.windll.kernel32 - k32.OpenProcess.restype = ctypes.c_void_p - k32.WaitForSingleObject.restype = ctypes.c_uint - k32.GetLastError.restype = ctypes.c_uint - h = k32.OpenProcess(0x1000 | 0x100000, False, int(p)) - if not h: - return k32.GetLastError() != 87 - try: - return k32.WaitForSingleObject(h, 0) == 0x102 - finally: - k32.CloseHandle(h) - try: - os.kill(int(p), 0) - return True - except ProcessLookupError: - return False - except PermissionError: - return True - except OSError: - return False - - while time.monotonic() < deadline: - if not _alive(pid): - break - time.sleep(0.2) - subprocess.Popen( - cmd, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - creationflags=windows_detach_flags_without_breakaway(), - ) - """ - ).strip() - from tools.environments.local import build_subprocess_env - watcher_env = build_subprocess_env(scrub_secrets=False, inherit_profile_home=True) - # The watcher must not inherit the gateway marker, else `hermes gateway restart` refuses to - # run (self-restart loop guard) and the gateway stays stopped. - watcher_env.pop("_HERMES_GATEWAY", None) - project_root = Path(__file__).resolve().parent.parent - # Console python under CREATE_NO_WINDOW owns one hidden console inherited by the restart - # child, so nothing flashes. Do NOT swap in pythonw.exe — a console-less watcher forces - # every console-subsystem descendant to allocate a visible conhost. - watcher_python = sys.executable - venv_dir = Path(watcher_env.get("VIRTUAL_ENV") or project_root / "venv") - site_packages = venv_dir / "Lib" / "site-packages" - if site_packages.exists(): - watcher_env["VIRTUAL_ENV"] = str(venv_dir) - pythonpath = [str(project_root), str(site_packages)] - if watcher_env.get("PYTHONPATH"): - pythonpath.append(watcher_env["PYTHONPATH"]) - watcher_env["PYTHONPATH"] = os.pathsep.join(dict.fromkeys(pythonpath)) - watcher_argv = [ - watcher_python, - "-c", - watcher, - str(current_pid), - str(restart_after_s), - *cmd_argv, - ] - # The watcher must break away from any job object the parent CLI lives in (Desktop - # wrappers, Windows Terminal, schtasks), else it is reaped when the CLI exits and the - # gateway never respawns. windows_detach_popen_kwargs() sets CREATE_BREAKAWAY_FROM_JOB, - # but a job without JOB_OBJECT_LIMIT_BREAKAWAY_OK rejects it (ERROR_ACCESS_DENIED as - # OSError); retry once without the bit, preserving argv and the scrubbed watcher_env. - try: - subprocess.Popen( - watcher_argv, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - env=watcher_env, - **windows_detach_popen_kwargs(), - ) - except OSError: - try: - subprocess.Popen( - watcher_argv, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - env=watcher_env, - creationflags=windows_detach_flags_without_breakaway(), - ) - except OSError as exc: - # Both spawns failed. Log only the interpreter basename and numeric errno — never - # argv, env, watcher source, or str(exc) (may carry a full path) — and return. - winerror = getattr(exc, "winerror", None) - error_code = winerror if winerror is not None else exc.errno - error_field = "winerror" if winerror is not None else "errno" - logger.warning( - "Detached restart watcher was not started after the " - "no-breakaway retry (%s; %s=%r). The gateway will not " - "be respawned by this restart attempt.", - os.path.basename(watcher_python), - error_field, - error_code, - ) + GatewayShutdownMixin._spawn_windows_restart_watcher(hermes_cmd, current_pid, restart_after_s) return - cmd = " ".join(shlex.quote(part) for part in hermes_cmd) shell_cmd = ( f"deadline=$(( $(date +%s) + {int(restart_after_s)} )); " f"while kill -0 {current_pid} 2>/dev/null && [ $(date +%s) -lt $deadline ]; do sleep 0.2; done; " f"{cmd} gateway restart" ) - # Same marker scrub as the Windows watcher: an inherited _HERMES_GATEWAY=1 makes the CLI's - # self-restart loop guard refuse silently (DEVNULL), so the gateway stops and never comes back. - from tools.environments.local import build_subprocess_env - watcher_env = build_subprocess_env(scrub_secrets=False, inherit_profile_home=True) - watcher_env.pop("_HERMES_GATEWAY", None) setsid_bin = shutil.which("setsid") - if setsid_bin: - subprocess.Popen( - [setsid_bin, "bash", "-lc", shell_cmd], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - env=watcher_env, - start_new_session=True, - ) - else: - subprocess.Popen( - ["bash", "-lc", shell_cmd], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - env=watcher_env, - start_new_session=True, - ) + argv = [setsid_bin, "bash", "-lc", shell_cmd] if setsid_bin else ["bash", "-lc", shell_cmd] + subprocess.Popen( + argv, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + env=GatewayShutdownMixin._restart_watcher_env(), start_new_session=True, + ) def _wedged_agent_count(self) -> int: - """Count running chat agents already past the inactivity timeout. + """Running chat agents with no activity for ``agent.gateway_timeout`` (0 when disabled). - No activity (API bytes, tool progress) for ``agent.gateway_timeout`` = wedged (the turn reaper's - threshold). Returns 0 when the timeout is disabled (the after-turn cap still bounds the wait). - Cron/API-server work has no activity clock and pending sentinels are brand-new, so neither - counts. Fail-open per agent: an unreadable activity summary means "not wedged". + Cron/API work has no activity clock and pending sentinels are brand-new, so neither counts; + an unreadable activity summary means "not wedged". """ from gateway.run import _AGENT_PENDING_SENTINEL, _float_env timeout = _float_env("HERMES_AGENT_TIMEOUT", 1800) if timeout <= 0: return 0 - wedged = 0 - for agent in list((getattr(self, "_running_agents", None) or {}).values()): - if agent is None or agent is _AGENT_PENDING_SENTINEL: - continue + + def _idle_seconds(agent: Any) -> Optional[float]: summary_fn = getattr(agent, "get_activity_summary", None) if not callable(summary_fn): - continue + return None try: summary = summary_fn() - if not isinstance(summary, dict): - continue - idle = float(summary.get("seconds_since_activity", 0.0)) + return float(summary.get("seconds_since_activity", 0.0)) if isinstance(summary, dict) else None except Exception: - continue - if idle >= timeout: - wedged += 1 - return wedged + return None + + return sum( + 1 + for agent in list((getattr(self, "_running_agents", None) or {}).values()) + if agent is not None and agent is not _AGENT_PENDING_SENTINEL + and (idle := _idle_seconds(agent)) is not None and idle >= timeout + ) def _awaitable_work_count(self) -> int: """Active work minus wedged turns — what the restart wait waits on.""" return max(0, self._active_work_count() - self._wedged_agent_count()) async def _await_active_work_before_restart(self) -> bool: - """Wait for in-flight work to finish before entering ``stop()``. + """Wait for in-flight work before ``stop()`` so the requesting turn isn't force-interrupted. - Calling ``stop()`` immediately would fold the requesting turn into the drain set and - force-interrupt it at ``restart_drain_timeout``; instead refuse new turns, wait for active - agents/cron/api work to reach zero, then ``stop()`` an idle gateway. Wedged turns - (``_wedged_agent_count``) are excluded — restart is the remedy, so ``stop()``'s drain - interrupts them. Returns True when drained to zero, False when the safety cap elapsed or - only wedged work remains (caller proceeds to ``stop()``). + Wedged turns are excluded (restart is their remedy). True when drained to zero, False when the + cap elapsed or only wedged work remains (caller proceeds to ``stop()``). """ active = self._active_work_count() if active <= 0: return True - - awaitable = self._awaitable_work_count() - if awaitable <= 0: + if self._awaitable_work_count() <= 0: logger.warning( "Restart requested with %d active work unit(s), all wedged " "past the inactivity timeout; skipping the after-turn wait " - "and proceeding to stop()/drain which will interrupt them", - active, + "and proceeding to stop()/drain which will interrupt them", active, ) return False - timeout = float(getattr(self, "_restart_after_turn_timeout", 0.0) or 0.0) if timeout <= 0: logger.info( "Restart requested with %d active work unit(s); " - "restart_after_turn_timeout=0 — entering stop()/drain immediately", - active, + "restart_after_turn_timeout=0 — entering stop()/drain immediately", active, ) return False - logger.info( "Restart requested with %d active work unit(s); " "deferring stop() until they finish (cap=%.0fs) so in-flight " - "turns are not amputated (#77184)", - active, - timeout, + "turns are not amputated (#77184)", active, timeout, ) with suppress(Exception): self._update_runtime_status("draining") - loop = asyncio.get_running_loop() deadline = loop.time() + timeout last_status_at = 0.0 @@ -1569,36 +1224,26 @@ class GatewayShutdownMixin: logger.warning( "Restart after-turn wait timed out after %.0fs with %d " "still active; proceeding to stop()/drain which may " - "interrupt remaining work (#77184)", - timeout, - self._active_work_count(), + "interrupt remaining work (#77184)", timeout, self._active_work_count(), ) return False if (now - last_status_at) >= 30.0: logger.info( "Restart deferred: waiting on %d active work unit(s) " "(%d wedged and excluded; %.0fs remaining before force drain)", - self._awaitable_work_count(), - self._wedged_agent_count(), - deadline - now, + self._awaitable_work_count(), self._wedged_agent_count(), deadline - now, ) with suppress(Exception): self._update_runtime_status("draining") last_status_at = now await asyncio.sleep(0.1) - if self._active_work_count() > 0: logger.warning( "Restart deferred wait: %d wedged work unit(s) remain; " - "proceeding to stop()/drain which will interrupt them", - self._active_work_count(), + "proceeding to stop()/drain which will interrupt them", self._active_work_count(), ) return False - - logger.info( - "Restart deferred wait complete — active work drained; " - "proceeding to stop()" - ) + logger.info("Restart deferred wait complete — active work drained; proceeding to stop()") return True def request_restart(self, *, detached: bool = False, via_service: bool = False) -> bool: @@ -1624,9 +1269,8 @@ class GatewayShutdownMixin: await asyncio.sleep(0.05) await self.stop(restart=True, detached_restart=detached, service_restart=via_service) - # Do NOT add _run_restart to _background_tasks: _stop_impl cancels every entry there, which - # would cancel it while awaiting _stop_task and propagate CancelledError into _stop_impl, - # skipping _shutdown_event.set() / _exit_code = 75. Keep a strong ref in self._restart_task. + # NOT in _background_tasks: _stop_impl cancels every entry there, which would propagate + # CancelledError into _stop_impl and skip _shutdown_event.set() / _exit_code = 75. self._restart_task = asyncio.create_task(_run_restart()) return True @@ -1636,9 +1280,7 @@ class GatewayShutdownMixin: return False if self._systemd_watchdog is not None: return True - from gateway.systemd_notify import SystemdWatchdog - watchdog = SystemdWatchdog(config_enabled=True) if not watchdog.start(): return False @@ -1654,328 +1296,200 @@ class GatewayShutdownMixin: self._systemd_watchdog = None await watchdog.stop() + # stop() phases. Invoked as ``GatewayRunner._stop_(self, ctx)`` so shutdown-path tests + # can drive them from bare doubles that are not GatewayRunner instances. @staticmethod def _stop_kill_tool_subprocesses(phase: str) -> list: - """Kill tool subprocesses + tear down terminal envs + browsers. + """Kill tool subprocesses + terminal envs + browsers; returns cron job IDs marked interrupted. - Returns the cron job IDs marked interrupted so the caller can notify owners while - adapters are still up. Called twice: eagerly after a drain timeout forces interrupt - (reclaim children before systemd SIGKILLs) and as a final catch-all in _stop_impl(). - Best-effort; exceptions swallowed so one subsystem cannot block the rest. + Called twice: after a drain timeout (reclaim children before systemd SIGKILLs) and as a final + catch-all. Best-effort; one failing subsystem cannot block the rest. """ - try: + + def _step(label: str, fn: Callable[[], Any]) -> Any: + try: + return fn() + except Exception as _e: + logger.debug("%s (%s) error: %s", label, phase, _e) + return None + + def _kill_processes() -> None: from tools.process_registry import process_registry _killed = process_registry.kill_all() if _killed: - logger.info( - "Shutdown (%s): killed %d tool subprocess(es)", - phase, _killed, - ) - except Exception as _e: - logger.debug("process_registry.kill_all (%s) error: %s", phase, _e) - _marked_cron_jobs: list = [] - try: + logger.info("Shutdown (%s): killed %d tool subprocess(es)", phase, _killed) + + def _mark_cron_interrupted() -> list: # kill_all() is a global sweep, so any cron job dispatched right now lost its tool # subprocess; its agent thread may still emit a plausible response from truncated # output. Mark the run interrupted so it can never be reported as success. from cron.scheduler import mark_running_jobs_interrupted - _interrupted = _marked_cron_jobs = mark_running_jobs_interrupted( - f"Gateway shutdown ({phase}) killed the job's tool " - "subprocess before the run finished." + _interrupted = mark_running_jobs_interrupted( + f"Gateway shutdown ({phase}) killed the job's tool subprocess before the run finished." ) if _interrupted: logger.warning( "Shutdown (%s): marked %d in-flight cron job(s) interrupted: %s", phase, len(_interrupted), ", ".join(_interrupted), ) - except Exception as _e: - logger.debug("mark_running_jobs_interrupted (%s) error: %s", phase, _e) - try: + return _interrupted + + def _interrupt_delegations() -> None: from tools.async_delegation import interrupt_all as _interrupt_async _async_n = _interrupt_async(reason=f"gateway shutdown ({phase})") if _async_n: - logger.info( - "Shutdown (%s): interrupted %d background delegation(s)", - phase, _async_n, - ) - except Exception as _e: - logger.debug("async interrupt_all (%s) error: %s", phase, _e) - try: + logger.info("Shutdown (%s): interrupted %d background delegation(s)", phase, _async_n) + + def _cleanup_environments() -> None: from tools.terminal_tool import cleanup_all_environments cleanup_all_environments() - except Exception as _e: - logger.debug("cleanup_all_environments (%s) error: %s", phase, _e) - try: + + def _cleanup_browsers() -> None: from tools.browser_tool import cleanup_all_browsers cleanup_all_browsers() - except Exception as _e: - logger.debug("cleanup_all_browsers (%s) error: %s", phase, _e) + + _step("process_registry.kill_all", _kill_processes) + _marked_cron_jobs = _step("mark_running_jobs_interrupted", _mark_cron_interrupted) or [] + _step("async interrupt_all", _interrupt_delegations) + _step("cleanup_all_environments", _cleanup_environments) + _step("cleanup_all_browsers", _cleanup_browsers) return _marked_cron_jobs - async def _stop_begin_teardown( - self, _stop_started_at_box: dict - ) -> Tuple[Callable[[], int], Callable[[], float]]: - """Flag teardown, stop room worker/watchdog, notify sessions. Returns the phase clocks.""" - # Shutdown-path tests and third-party runner doubles may only - # implement the older drain-count surface. - _deferred_worker_count = getattr( - self, - "_active_deferred_agent_worker_count", - lambda: 0, - ) - logger.info( - "Stopping gateway%s...", - " for restart" if self._restart_requested else "", - ) - _stop_started_at = time.monotonic() - _stop_started_at_box["t"] = _stop_started_at - - def _phase_elapsed() -> float: - return time.monotonic() - _stop_started_at - + async def _stop_begin_teardown(self, ctx: "GatewayShutdownMixin._StopContext") -> None: + """Flag teardown, stop room worker/watchdog, notify sessions.""" + logger.info("Stopping gateway%s...", " for restart" if self._restart_requested else "") + ctx.started_at = time.monotonic() self._running = False self._clear_plugin_message_injector() self._draining = True - + # getattr-guards: shutdown-path test doubles may lack the room worker / systemd watchdog. stop_room_worker = getattr(self, "_stop_hosted_room_worker", None) if callable(stop_room_worker): try: - stopped = await stop_room_worker(timeout=5.0) - if not stopped: + if not await stop_room_worker(timeout=5.0): logger.warning( - "Group Chat worker is still settling durable work; " - "the next gateway start will recover it" + "Group Chat worker is still settling durable work; the next gateway start will recover it" ) except Exception: logger.warning( - "Group Chat worker could not stop cleanly; the next gateway " - "start will recover durable work", + "Group Chat worker could not stop cleanly; the next gateway start will recover durable work", exc_info=True, ) - stop_watchdog = getattr(self, "_stop_systemd_watchdog", None) if callable(stop_watchdog): await stop_watchdog() - await self._cancel_secondary_profile_reconnect_tasks() - - # Notify all chats with active agents BEFORE draining. - # Adapters are still connected here, so messages can be sent. + # Notify all chats with active agents BEFORE draining — adapters are still connected here. await self._notify_active_sessions_of_shutdown() - logger.info( - "Shutdown phase: notify_active_sessions done at +%.2fs", - _phase_elapsed(), + logger.info("Shutdown phase: notify_active_sessions done at +%.2fs", ctx.elapsed()) + + async def _stop_drain_active_work(self, timeout: float, ctx: "GatewayShutdownMixin._StopContext") -> None: + """Pre-mark resume_pending, drain agents/cron/API work into ``ctx``.""" + from gateway.run import GatewayRunner + # Pre-mark resume_pending BEFORE the drain: if the service manager kills us mid-drain, the + # durable marker already lets the next boot recover the sessions. + _pre_drain_keys = await GatewayRunner._mark_running_sessions_resume_pending( + self, "pre-drain mark_resume_pending" ) - return _deferred_worker_count, _phase_elapsed - - async def _stop_drain_active_work( - self, - timeout: float, - _deferred_worker_count: Callable[[], int], - _phase_elapsed: Callable[[], float], - ) -> Tuple[dict, bool, float]: - """Pre-mark resume_pending, drain agents/cron/API work. Returns (active_agents, timed_out, drain_elapsed).""" - from gateway.run import _AGENT_PENDING_SENTINEL - # Pre-mark sessions resume_pending BEFORE the drain wait: if the service manager kills - # the process mid-drain, the durable marker already lets the next boot recover them. - _pre_drain_keys: list[str] = [] - for _sk, _agent in list(self._running_agents.items()): - if _agent is _AGENT_PENDING_SENTINEL: - continue - try: - await self.async_session_store.mark_resume_pending( - _sk, - "restart_timeout" if self._restart_requested else "shutdown_timeout", - ) - _pre_drain_keys.append(_sk) - except Exception as _e: - logger.debug("pre-drain mark_resume_pending failed for %s: %s", _sk, _e) - _cron_at_start = self._active_cron_job_count() _api_at_start = self._active_api_run_count() - _deferred_at_start = _deferred_worker_count() - # In-flight cron work gets its own floor, clamped to the watchdog leash so the extra - # wait never costs the post-drain cleanup window. getattr-guard: shutdown-path tests - # drive _stop_impl_body from bare doubles (not GatewayRunner) lacking the class default. - _cron_drain_cfg = getattr( - self, "_cron_drain_timeout", DEFAULT_GATEWAY_CRON_DRAIN_TIMEOUT - ) + _deferred_at_start = ctx.deferred_count() + # Cron floor clamped to the watchdog leash so it never costs the post-drain cleanup window. + # getattr-guard: shutdown-path tests drive the phases from bare doubles lacking the class default. + _cron_drain_cfg = getattr(self, "_cron_drain_timeout", DEFAULT_GATEWAY_CRON_DRAIN_TIMEOUT) _cron_timeout = resolve_cron_drain_budget( - timeout, - _cron_drain_cfg, - watchdog_delay=resolve_shutdown_watchdog_delay(timeout), - elapsed=_phase_elapsed(), + timeout, _cron_drain_cfg, watchdog_delay=resolve_shutdown_watchdog_delay(timeout), + elapsed=ctx.elapsed(), ) if _cron_at_start and _cron_timeout > timeout: logger.info( "Shutdown drain: %d in-flight cron job(s) — waiting up to " "%.0fs for them (cron_drain_timeout=%.0fs, " "restart_drain_timeout=%.0fs)", - _cron_at_start, - _cron_timeout, - _cron_drain_cfg, - timeout, + _cron_at_start, _cron_timeout, _cron_drain_cfg, timeout, ) _drain_started_at = time.monotonic() - active_agents, timed_out = await self._drain_active_agents( - timeout, _cron_timeout - ) - _drain_elapsed = time.monotonic() - _drain_started_at + ctx.active_agents, ctx.timed_out = await self._drain_active_agents(timeout, _cron_timeout) + ctx.drain_elapsed = time.monotonic() - _drain_started_at logger.info( "Shutdown phase: drain done at +%.2fs (drain took %.2fs, " "timed_out=%s, active_at_start=%d, active_now=%d, " "cron_at_start=%d, cron_now=%d, " "api_at_start=%d, api_now=%d, " - "deferred_at_start=%d, deferred_now=%d)", - _phase_elapsed(), - _drain_elapsed, - timed_out, - len(active_agents), - self._running_agent_count(), - _cron_at_start, - self._active_cron_job_count(), - _api_at_start, - self._active_api_run_count(), - _deferred_at_start, - _deferred_worker_count(), + "deferred_at_start=%d, deferred_now=%d)", ctx.elapsed(), ctx.drain_elapsed, + ctx.timed_out, len(ctx.active_agents), self._running_agent_count(), _cron_at_start, + self._active_cron_job_count(), _api_at_start, self._active_api_run_count(), + _deferred_at_start, ctx.deferred_count(), ) + if ctx.timed_out: + return + # Graceful drain: clear the pre-drain resume_pending markers so sessions that finished + # during the drain window don't carry a stale flag. + for _sk in _pre_drain_keys: + if _sk not in self._running_agents: + try: + await self.async_session_store.clear_resume_pending(_sk) + except Exception as _e: + logger.debug("clear_resume_pending after drain failed for %s: %s", _sk, _e) - if not timed_out: - # Graceful drain: clear the pre-drain resume_pending markers so sessions that - # finished during the drain window don't carry a stale flag. - for _sk in _pre_drain_keys: - if _sk not in self._running_agents: - try: - await self.async_session_store.clear_resume_pending(_sk) - except Exception as _e: - logger.debug( - "clear_resume_pending after drain failed for %s: %s", - _sk, _e, - ) - return active_agents, timed_out, _drain_elapsed - - async def _stop_interrupt_remaining_work( - self, - _drain_elapsed: float, - _deferred_worker_count: Callable[[], int], - _phase_elapsed: Callable[[], float], - ) -> None: + async def _stop_interrupt_remaining_work(self, ctx: "GatewayShutdownMixin._StopContext") -> None: """Drain timed out: mark resume_pending, interrupt, settle, kill tool subprocesses, notify cron.""" - from gateway.run import ( - GatewayRunner, - _AGENT_PENDING_SENTINEL, - _INTERRUPT_REASON_GATEWAY_RESTART, - _INTERRUPT_REASON_GATEWAY_SHUTDOWN, - ) + from gateway.run import GatewayRunner logger.warning( "Gateway drain timed out after %.1fs with %d active agent(s), " "%d in-flight cron job(s), %d api_server run(s), and " "%d deferred agent worker(s); " - "interrupting remaining work.", - _drain_elapsed, - self._running_agent_count(), - self._active_cron_job_count(), - self._active_api_run_count(), - _deferred_worker_count(), + "interrupting remaining work.", ctx.drain_elapsed, self._running_agent_count(), + self._active_cron_job_count(), self._active_api_run_count(), ctx.deferred_count(), ) - # Mark forcibly-interrupted sessions resume_pending BEFORE interrupting, so the next - # message on the same session_key auto-resumes instead of being converted to a fresh - # session by suspend_recently_active(). Genuinely stuck sessions still escalate via - # ``.restart_failure_counts`` (threshold 3), which sets ``suspended=True`` and wins. - # - # Iterate self._running_agents (current), not the drain-start snapshot: sessions that - # finished cleanly during the drain would otherwise get a stray interruption note. - # Skip pending sentinels as _interrupt_running_agents() does — nothing has started. - _resume_reason = ( - "restart_timeout" if self._restart_requested else "shutdown_timeout" - ) - for _sk, _agent in list(self._running_agents.items()): - if _agent is _AGENT_PENDING_SENTINEL: - continue - try: - await self.async_session_store.mark_resume_pending(_sk, _resume_reason) - except Exception as _e: - logger.debug( - "mark_resume_pending failed for %s: %s", - _sk, _e, - ) - self._interrupt_running_agents( - _INTERRUPT_REASON_GATEWAY_RESTART if self._restart_requested else _INTERRUPT_REASON_GATEWAY_SHUTDOWN - ) - interrupt_grace_timeout = ( - GatewayRunner._post_interrupt_grace_timeout(self) - ) - interrupt_deadline = ( - asyncio.get_running_loop().time() + interrupt_grace_timeout - ) - logger.info( - "Shutdown phase: allowing %.1fs for interrupted agents to unwind", - interrupt_grace_timeout, - ) - # Wait on API-server work too: the interrupt is cooperative, and without this the - # settle window closes as soon as _running_agents is empty, so an API turn just asked - # to stop has its tool subprocesses killed below before it can unwind. - while ( - self._running_agents - or self._active_api_run_count() - or _deferred_worker_count() - ) and asyncio.get_running_loop().time() < interrupt_deadline: + # Mark forcibly-interrupted sessions resume_pending BEFORE interrupting so the next message + # auto-resumes instead of becoming a fresh session via suspend_recently_active(); stuck + # sessions still escalate via ``.restart_failure_counts`` (suspended=True wins). Uses the + # CURRENT _running_agents, not the drain-start snapshot, so cleanly finished sessions get no note. + await GatewayRunner._mark_running_sessions_resume_pending(self, "mark_resume_pending") + reason = GatewayRunner._shutdown_interrupt_reason(self) + self._interrupt_running_agents(reason) + interrupt_grace_timeout = GatewayRunner._post_interrupt_grace_timeout(self) + loop = asyncio.get_running_loop() + interrupt_deadline = loop.time() + interrupt_grace_timeout + logger.info("Shutdown phase: allowing %.1fs for interrupted agents to unwind", interrupt_grace_timeout) + + def _work_live() -> bool: + return bool(self._running_agents or self._active_api_run_count() or ctx.deferred_count()) + + # Wait on API-server work too: the interrupt is cooperative; closing the settle window as soon + # as _running_agents is empty would kill an API turn's tool subprocesses before it unwinds. + while _work_live() and loop.time() < interrupt_deadline: self._update_runtime_status("draining") await asyncio.sleep(0.1) - - # The interrupt fires once, but work can materialize AFTER it: a /v1/runs task enters - # _active_run_agents only when _create_agent returns, and a _AGENT_PENDING_SENTINEL - # entry is promoted by track_agent() on its own schedule. Re-signal anything still - # live so it gets a cooperative interrupt instead of a bare tool-subprocess kill. - if ( - self._running_agents - or self._active_api_run_count() - or _deferred_worker_count() - ): - self._interrupt_running_agents( - _INTERRUPT_REASON_GATEWAY_RESTART - if self._restart_requested - else _INTERRUPT_REASON_GATEWAY_SHUTDOWN - ) - logger.debug( - "Re-signaled interrupt for work still live at settle-window exit" - ) - - # Kill lingering tool subprocesses NOW, before adapter disconnect / DB close: under - # systemd (TimeoutStopSec ≈ drain_timeout + headroom) deferring risks the cgroup - # SIGKILL reaping orphaned children instead of us. The final catch-all still runs. + # Work can materialize AFTER the one-shot interrupt (a /v1/runs task registers only when + # _create_agent returns; a pending sentinel is promoted by track_agent() later). Re-signal so + # it gets a cooperative interrupt instead of a bare tool-subprocess kill. + if _work_live(): + self._interrupt_running_agents(reason) + logger.debug("Re-signaled interrupt for work still live at settle-window exit") + # Kill lingering tool subprocesses NOW, before adapter disconnect / DB close: under systemd + # (TimeoutStopSec ≈ drain_timeout + headroom) deferring risks the cgroup SIGKILL reaping them. _interrupted_cron_jobs = GatewayRunner._stop_kill_tool_subprocesses("post-interrupt") - logger.info( - "Shutdown phase: post-interrupt tool kill done at +%.2fs", - _phase_elapsed(), - ) - # Last window where the transport is still up. The cron worker whose run we just - # killed will try to deliver its own "interrupted" notice, but it gets there after - # the adapter teardown below and the message is lost. + logger.info("Shutdown phase: post-interrupt tool kill done at +%.2fs", ctx.elapsed()) + # Last window with the transport up: the cron worker's own "interrupted" notice arrives + # after adapter teardown and is lost. try: await self._notify_interrupted_cron_jobs(_interrupted_cron_jobs) except Exception as _e: logger.debug("Cron interrupt notification failed: %s", _e) - logger.info( - "Shutdown phase: cron interrupt notices done at +%.2fs", - _phase_elapsed(), - ) + logger.info("Shutdown phase: cron interrupt notices done at +%.2fs", ctx.elapsed()) - async def _stop_finalize_agents_and_adapters( - self, active_agents: dict, _phase_elapsed: Callable[[], float] - ) -> None: + async def _stop_finalize_agents_and_adapters(self, ctx: "GatewayShutdownMixin._StopContext") -> None: """Detached restart launch, agent finalization, idle-cache cleanup, adapter teardown.""" if self._restart_requested and self._restart_detached: try: await self._launch_detached_restart_command() except Exception as e: logger.error("Failed to launch detached gateway restart: %s", e) - - await self._finalize_shutdown_agents(active_agents) - - # Also shut down memory providers on idle cached agents. _finalize_shutdown_agents only - # handles agents that were mid-turn at drain time; the _agent_cache may still hold idle - # agents whose MemoryProviders never received on_session_end(). + await self._finalize_shutdown_agents(ctx.active_agents) + # Idle cached agents too: _finalize_shutdown_agents only covers agents mid-turn at drain + # time, but _agent_cache may hold agents whose MemoryProviders never got on_session_end(). _cache_lock = getattr(self, "_agent_cache_lock", None) _cache = getattr(self, "_agent_cache", None) if _cache_lock is not None and _cache is not None: @@ -1983,222 +1497,149 @@ class GatewayShutdownMixin: _idle_agents = list(_cache.values()) _cache.clear() for _entry in _idle_agents: - _agent = ( - _entry[0] if isinstance(_entry, tuple) else _entry - ) # Bounded + off-loop so a wedged memory provider can't hang shutdown forever # (this path is why SIGTERM once failed to kill the process). await self._cleanup_agent_resources_off_loop( - _agent, context="shutdown idle-cache" + _entry[0] if isinstance(_entry, tuple) else _entry, context="shutdown idle-cache" ) - - # Completion flush tasks can be sleeping in their fan-in window or blocked in adapter - # delivery. Cancel and await them while adapters are still alive so every watcher - # receives a retryable result before platform teardown begins. - cancel_completion_batches = getattr( - self, "_cancel_process_completion_batch_tasks", None - ) + # Cancel + await completion flush tasks (fan-in sleep / adapter delivery) while adapters are + # still alive so every watcher gets a retryable result before platform teardown. + cancel_completion_batches = getattr(self, "_cancel_process_completion_batch_tasks", None) if cancel_completion_batches is not None: await cancel_completion_batches() - for platform, adapter in list(self.adapters.items()): await self._bounded_adapter_teardown(adapter, platform) - # Disconnect secondary-profile adapters (multiplex mode). for _prof, _amap in list(getattr(self, "_profile_adapters", {}).items()): for platform, adapter in list(_amap.items()): - await self._bounded_adapter_teardown( - adapter, platform, profile=_prof - ) + await self._bounded_adapter_teardown(adapter, platform, profile=_prof) _amap.clear() if hasattr(self, "_profile_adapters"): self._profile_adapters.clear() - logger.info( - "Shutdown phase: all adapters disconnected at +%.2fs", - _phase_elapsed(), - ) + logger.info("Shutdown phase: all adapters disconnected at +%.2fs", ctx.elapsed()) - def _stop_release_runtime_state(self, _phase_elapsed: Callable[[], float]) -> None: + def _stop_release_runtime_state(self, ctx: "GatewayShutdownMixin._StopContext") -> None: """Cancel background tasks, flush pending messages, clear per-session state, final tool kill.""" from gateway.run import GatewayRunner for _task in list(self._background_tasks): - if _task is self._stop_task: - continue - if _task is self._restart_task: - # The restart orchestration task is awaiting _stop_task right now; cancelling it - # would propagate CancelledError into this _stop_impl and skip - # _shutdown_event.set() / _exit_code = 75. It self-terminates anyway. + # _restart_task is awaiting _stop_task right now; cancelling it would propagate + # CancelledError into _stop_impl and skip _shutdown_event.set(). It self-terminates anyway. + if _task is self._stop_task or _task is self._restart_task: continue _task.cancel() self._background_tasks.clear() - self.adapters.clear() for _session_key in list(self._running_agents): self._release_running_agent_state(_session_key) # Flush pending messages to disk before clearing: under FTS5 corruption the in-memory # pending text is the only surviving copy; clearing unflushed loses it permanently. - try: + with suppress(Exception): from gateway.shutdown_flush import flush_pending_to_file flush_pending_to_file(dict(self._pending_messages), reason="shutdown") - except Exception: - pass # The FIFO tail lives in SessionState.conversation.queued_events, not the slot dict # above — flush it too or every follow-up parked in overflow at restart time is lost. - try: + with suppress(Exception): from gateway.shutdown_flush import flush_overflow_to_file flush_overflow_to_file( - { - _k: list(_v) - for _k, _v in dict(getattr(self, "_queued_events", None) or {}).items() - if _v - }, + {_k: list(_v) for _k, _v in dict(getattr(self, "_queued_events", None) or {}).items() if _v}, reason="shutdown", ) - except Exception: - pass - # On the real runner these are live SessionState views whose clear() resets one field - # per session — never a wholesale dict swap, so a concurrent writer on another session - # can't lose its entry. Test fakes borrowing _stop_impl keep plain dicts. + # On the real runner these are live SessionState views whose clear() resets one field per + # session (never a wholesale dict swap that could lose a concurrent writer's entry). self._running_agents.clear() self._running_agents_ts.clear() if hasattr(self, "_active_session_leases"): self._active_session_leases.clear() self._pending_messages.clear() self._pending_approvals.clear() - if hasattr(self, '_busy_ack_ts'): + if hasattr(self, "_busy_ack_ts"): self._busy_ack_ts.clear() self._shutdown_event.set() - # Global catch-all subprocess kill (safe to repeat): covers the graceful path and # anything respawned since the drain-timeout path's post-interrupt kill. GatewayRunner._stop_kill_tool_subprocesses("final-cleanup") - logger.info( - "Shutdown phase: final-cleanup tool kill done at +%.2fs", - _phase_elapsed(), - ) - - # Reap the process-global auxiliary-client cache once at the end of teardown. Per-turn - # cleanup misses clients bound to worker-thread loops that died with their executor - # (cron ticks); without this sweep async httpx transports accumulate until EMFILE. + logger.info("Shutdown phase: final-cleanup tool kill done at +%.2fs", ctx.elapsed()) + # Reap the process-global auxiliary-client cache: per-turn cleanup misses clients bound to + # worker-thread loops that died with their executor (cron ticks) → httpx transports leak to EMFILE. try: from agent.auxiliary_client import shutdown_cached_clients shutdown_cached_clients() except Exception as _e: logger.debug("shutdown_cached_clients error: %s", _e) - def _stop_quiesce_and_close_session_dbs( - self, timeout: float, _phase_elapsed: Callable[[], float] - ) -> None: + def _stop_quiesce_and_close_session_dbs(self, timeout: float, ctx: "GatewayShutdownMixin._StopContext") -> None: """Quiesce the executor, then close SessionDB handles only if no worker is still live.""" from gateway.run import GatewayRunner, _EXECUTOR_QUIESCE_TIMEOUT - # Quiesce the gateway thread pool BEFORE the session databases are closed. Running it - # after the close left two holes: (a) ``_executor_closing`` was still False, so any - # coroutine reaching ``_run_in_executor_with_context`` minted a fresh pool and ran more - # blocking DB work against just-closed handles; (b) cancelling ``self._background_tasks`` - # does not stop a ``run_in_executor`` future that already started — the task dies, the - # worker keeps writing. Either way a write lands after ``SessionDB.close()`` has - # checkpointed the WAL and let SQLite unlink the sidecar; the late write silently - # reopens the handle and mints a fresh WAL generation behind that checkpoint, so - # teardown checkpoints the same file twice from an unaccounted connection - # (close-time page-write corruption / split WAL generation). - # The wait is bounded and clamped to what is left of the shutdown watchdog leash - # (minus a second for the close itself), so a stuck worker can never cost us the - # post-close cleanup window. + # Quiesce the thread pool BEFORE closing session DBs: otherwise a coroutine can mint a fresh + # pool (``_executor_closing`` still False) or an already-started run_in_executor future keeps + # writing after ``SessionDB.close()`` checkpointed the WAL — the late write reopens the handle + # and mints a split WAL generation (close-time corruption). Bounded and clamped to what is + # left of the watchdog leash minus a second for the close itself. _exec_quiesce_budget = max( - 0.0, - min( - _EXECUTOR_QUIESCE_TIMEOUT, - resolve_shutdown_watchdog_delay(timeout) - - _phase_elapsed() - - 1.0, - ), - ) - _exec_live = GatewayRunner._shutdown_executor( - self, drain_timeout=_exec_quiesce_budget + 0.0, min(_EXECUTOR_QUIESCE_TIMEOUT, resolve_shutdown_watchdog_delay(timeout) - ctx.elapsed() - 1.0), ) + _exec_live = GatewayRunner._shutdown_executor(self, drain_timeout=_exec_quiesce_budget) if _exec_live: - # A live worker can still be mid-write against a SessionDB - # handle. Checkpointing/closing it now is exactly the - # sequence that produced the wrong-page-number corruption in - # #101093, so the close path below is skipped entirely - # rather than raced — the handle is left open for SQLite to - # recover from its own WAL on the next open, which is a - # transient "database is locked" on an immediate --replace - # at worst, not a corrupt file. + # A live worker may be mid-write: closing now is the exact #101093 corruption sequence, + # so skip the close and let SQLite recover from its WAL on next open (a transient + # "database is locked" on an immediate --replace at worst, not a corrupt file). logger.warning( "Shutdown phase: %d executor worker(s) still running after " "a %.2fs quiesce — skipping the SessionDB close/checkpoint " "to avoid racing a live write (#101093); handles are left " - "open for SQLite to recover on next open", - _exec_live, - _exec_quiesce_budget, - ) - else: - logger.info( - "Shutdown phase: executor quiesced at +%.2fs", - _phase_elapsed(), + "open for SQLite to recover on next open", _exec_live, _exec_quiesce_budget, ) + return + logger.info("Shutdown phase: executor quiesced at +%.2fs", ctx.elapsed()) - # Close SQLite session DBs so the WAL lock is released; otherwise --replace leaves the old - # connection holding it until exit and the new gateway gets 'database is locked'. - # ``_session_db`` is an AsyncSessionDB facade — unwrap; ``session_store`` holds ``_db``. - _self_db = getattr(self, "_session_db", None) - _self_db = getattr(_self_db, "_db", _self_db) - for _db in (_self_db, getattr(getattr(self, "session_store", None), "_db", None)): - if _db is None or not hasattr(_db, "close"): - continue - try: - _db.close() - except Exception as _e: - logger.debug("SessionDB close error: %s", _e) - # A multiplexed session_store caches one SessionDB per profile; ``_db`` above only covered - # the root scope. Sweep the rest so secondary WAL locks are released before --replace. - _sweep = getattr( - getattr(self, "session_store", None), "close_all_db_handles", None - ) - if _sweep is not None: - try: - _sweep() - except Exception as _e: - logger.debug("SessionDB handle sweep error: %s", _e) - # Same sweep for the runner's own per-profile session_search - # handles (slash commands resolve them under profile scopes). + def _step(label: str, fn: Callable[[], Any]) -> None: try: - GatewayRunner.close_all_session_db_handles(self) + fn() except Exception as _e: - logger.debug("Runner SessionDB handle sweep error: %s", _e) - # Final sweep: close shared SessionDB instances still held by the process-wide registry - # (tools, cron, mirror, etc. opened via get_shared_session_db but not released above). - try: - from hermes_state import close_shared_session_dbs - closed = close_shared_session_dbs() - if closed: - logger.debug("Closed %d shared SessionDB instance(s) at shutdown", closed) - except Exception as _e: - logger.debug("Shared SessionDB close error: %s", _e) - logger.info( - "Shutdown phase: SessionDB close done at +%.2fs", - _phase_elapsed(), - ) + logger.debug("%s: %s", label, _e) - def _stop_persist_exit_state( - self, timed_out: bool, active_agents: dict, _phase_elapsed: Callable[[], float] - ) -> None: + # Close SQLite session DBs so the WAL lock is released; otherwise --replace leaves the old + # connection holding it until exit and the new gateway gets 'database is locked'. + # ``_session_db`` is an AsyncSessionDB facade — unwrap; ``session_store`` holds ``_db``. + _self_db = getattr(self, "_session_db", None) + _self_db = getattr(_self_db, "_db", _self_db) + store = getattr(self, "session_store", None) + for _db in (_self_db, getattr(store, "_db", None)): + if _db is not None and hasattr(_db, "close"): + _step("SessionDB close error", _db.close) + # A multiplexed session_store caches one SessionDB per profile; ``_db`` above only covered + # the root scope. Sweep the rest so secondary WAL locks are released before --replace. + _sweep = getattr(store, "close_all_db_handles", None) + if _sweep is not None: + _step("SessionDB handle sweep error", _sweep) + # Same sweep for the runner's own per-profile session_search handles (slash commands + # resolve them under profile scopes). + _step("Runner SessionDB handle sweep error", lambda: GatewayRunner.close_all_session_db_handles(self)) + + def _close_shared() -> None: + # Shared SessionDB instances still held by the process-wide registry (tools, cron, + # mirror, etc. opened via get_shared_session_db but not released above). + from hermes_state import close_shared_session_dbs + closed = close_shared_session_dbs() + if closed: + logger.debug("Closed %d shared SessionDB instance(s) at shutdown", closed) + + _step("Shared SessionDB close error", _close_shared) + logger.info("Shutdown phase: SessionDB close done at +%.2fs", ctx.elapsed()) + + def _stop_persist_exit_state(self, ctx: "GatewayShutdownMixin._StopContext") -> None: """PID/lock release, clean-shutdown marker, restart markers, terminal runtime status.""" from gateway.run import ( - _hermes_home, - _planned_restart_notification_path, - _shutdown_gateway_health_export, + _hermes_home, _planned_restart_notification_path, _shutdown_gateway_health_export, atomic_json_write, ) from gateway.status import remove_pid_file, release_gateway_runtime_lock remove_pid_file() release_gateway_runtime_lock() - # Clean-shutdown marker: suspend_recently_active() need only run after unexpected exits. # If the drain timed out and agents were force-interrupted, sessions may be half-finished # — skip the marker so the next startup suspends them. - if not timed_out: + if not ctx.timed_out: with suppress(Exception): (_hermes_home / ".clean_shutdown").touch() else: @@ -2207,12 +1648,10 @@ class GatewayShutdownMixin: "interrupted agents; next startup will suspend recently " "active sessions." ) - # Stuck-loop detection: the counter increments for sessions active at each restart; at # the threshold (3 consecutive) the next startup auto-suspends the session. - if active_agents: - self._increment_restart_failure_counts(set(active_agents.keys())) - + if ctx.active_agents: + self._increment_restart_failure_counts(set(ctx.active_agents.keys())) if self._restart_requested and self._restart_command_source is None: try: atomic_json_write( @@ -2226,19 +1665,15 @@ class GatewayShutdownMixin: ) except Exception as e: logger.debug("Failed to write planned restart notification marker: %s", e) - if self._restart_requested and self._restart_via_service: # Service manager owns restarts: exit 75 + ``RestartForceExitStatus=75`` has systemd # replace this process without a second helper racing the unit's stop/start job. self._exit_code = GATEWAY_SERVICE_RESTART_EXIT_CODE self._exit_reason = self._exit_reason or "Gateway restart requested" - self._draining = False - # Persist terminal gateway_state: "stopped" by default, but "running" on an UNEXPECTED - # external signal (s6 SIGTERM on docker restart, OOM-kill, kill) — container_boot.py - # only auto-starts gateways last seen "running", so "stopped"/"draining" after a routine - # recreate would leave channels dark. Operator stops write a planned-stop marker BEFORE - # signalling and persist "stopped"; a restart also persists "stopped". + # Terminal gateway_state: "stopped" by default, "running" on an UNEXPECTED signal (s6 SIGTERM + # on docker restart, OOM-kill) — container_boot.py only auto-starts gateways last seen + # "running". Operator stops write a planned-stop marker BEFORE signalling; restarts persist "stopped". if getattr(self, "_signal_initiated_shutdown", False) and not self._restart_requested: logger.info( "Gateway stopped by an unexpected signal — persisting " @@ -2249,19 +1684,64 @@ class GatewayShutdownMixin: else: self._update_runtime_status("stopped", self._exit_reason) _shutdown_gateway_health_export(self) - logger.info("Gateway stopped (total teardown %.2fs)", _phase_elapsed()) + logger.info("Gateway stopped (total teardown %.2fs)", ctx.elapsed()) + + def _shutdown_watchdog_snapshot(self, ctx: "GatewayShutdownMixin._StopContext") -> dict: + """State dumped by the thread-based shutdown watchdog when teardown hangs.""" + return { + "restart_requested": bool(self._restart_requested), + "draining": bool(self._draining), + "running": bool(self._running), + "active_agents": self._running_agent_count(), + "active_cron_jobs": self._active_cron_job_count(), + "active_api_runs": self._active_api_run_count(), + "active_deferred_agent_workers": ctx.deferred_count(), + "restart_drain_timeout": self._restart_drain_timeout, + "watchdog_delay_s": resolve_shutdown_watchdog_delay(self._restart_drain_timeout), + "phase_elapsed_s": ctx.elapsed() if ctx.started_at is not None else None, + } + + async def _stop_impl(self) -> None: + """Run every ``_stop_*`` phase under the thread-based shutdown watchdog.""" + from gateway.run import GatewayRunner + # Thread-based watchdog (asyncio timeouts cannot recover a frozen loop): if teardown never + # finishes within drain+grace it dumps faulthandler stacks and os._exit so KeepAlive/systemd + # can revive. Skipped under pytest (no delayed hard-exit in the worker). + _watchdog_done = threading.Event() + self._shutdown_watchdog_done = _watchdog_done + # Shutdown-path tests and third-party runner doubles may only implement the older + # drain-count surface (no deferred-worker counter). + ctx = GatewayShutdownMixin._StopContext( + deferred_count=getattr(self, "_active_deferred_agent_worker_count", lambda: 0) + ) + if not os.environ.get("PYTEST_CURRENT_TEST"): + arm_shutdown_watchdog( + resolve_shutdown_watchdog_delay(self._restart_drain_timeout), + done_event=_watchdog_done, + snapshot_fn=lambda: GatewayRunner._shutdown_watchdog_snapshot(self, ctx), + exit_code=1, + ) + try: + await GatewayRunner._stop_begin_teardown(self, ctx) + timeout = self._restart_drain_timeout + await GatewayRunner._stop_drain_active_work(self, timeout, ctx) + if ctx.timed_out: + await GatewayRunner._stop_interrupt_remaining_work(self, ctx) + await GatewayRunner._stop_finalize_agents_and_adapters(self, ctx) + GatewayRunner._stop_release_runtime_state(self, ctx) + GatewayRunner._stop_quiesce_and_close_session_dbs(self, timeout, ctx) + GatewayRunner._stop_persist_exit_state(self, ctx) + finally: + _watchdog_done.set() async def stop( - self, - *, - restart: bool = False, - detached_restart: bool = False, + self, *, restart: bool = False, detached_restart: bool = False, service_restart: bool = False, ) -> None: """Stop the gateway and disconnect all adapters.""" from gateway.run import GatewayRunner - # getattr-guard: shutdown-path tests build bare runners via - # object.__new__ that lack the liveness-guard machinery. + # getattr-guard: shutdown-path tests build bare runners via object.__new__ that lack the + # liveness-guard machinery. _stop_guards = getattr(self, "_stop_loop_liveness_guards", None) if callable(_stop_guards): _stop_guards() @@ -2272,75 +1752,7 @@ class GatewayShutdownMixin: if self._stop_task is not None: await self._stop_task return - - async def _stop_impl() -> None: - # Thread-based shutdown watchdog: asyncio timeouts cannot recover a frozen loop. Arm a - # plain OS thread at the start of stop(); if teardown never finishes within drain+grace - # it dumps faulthandler stacks and os._exit so KeepAlive/systemd can revive. Skipped - # under pytest so stop()-driving tests don't get a delayed hard-exit in the worker. - _watchdog_done = threading.Event() - self._shutdown_watchdog_done = _watchdog_done - _stop_started_at_box: dict[str, float] = {} - - def _shutdown_watchdog_snapshot() -> dict: - started = _stop_started_at_box.get("t") - return { - "restart_requested": bool(self._restart_requested), - "draining": bool(self._draining), - "running": bool(self._running), - "active_agents": self._running_agent_count(), - "active_cron_jobs": self._active_cron_job_count(), - "active_api_runs": self._active_api_run_count(), - "active_deferred_agent_workers": getattr( - self, - "_active_deferred_agent_worker_count", - lambda: 0, - )(), - "restart_drain_timeout": self._restart_drain_timeout, - "watchdog_delay_s": resolve_shutdown_watchdog_delay( - self._restart_drain_timeout - ), - "phase_elapsed_s": ( - time.monotonic() - started if started is not None else None - ), - } - - if not os.environ.get("PYTEST_CURRENT_TEST"): - arm_shutdown_watchdog( - resolve_shutdown_watchdog_delay(self._restart_drain_timeout), - done_event=_watchdog_done, - snapshot_fn=_shutdown_watchdog_snapshot, - exit_code=1, - ) - - try: - await _stop_impl_body(_stop_started_at_box) - finally: - _watchdog_done.set() - - async def _stop_impl_body(_stop_started_at_box) -> None: - _deferred_worker_count, _phase_elapsed = await GatewayRunner._stop_begin_teardown( - self, _stop_started_at_box - ) - - timeout = self._restart_drain_timeout - active_agents, timed_out, _drain_elapsed = await GatewayRunner._stop_drain_active_work( - self, timeout, _deferred_worker_count, _phase_elapsed - ) - - if timed_out: - await GatewayRunner._stop_interrupt_remaining_work( - self, _drain_elapsed, _deferred_worker_count, _phase_elapsed - ) - - await GatewayRunner._stop_finalize_agents_and_adapters( - self, active_agents, _phase_elapsed - ) - GatewayRunner._stop_release_runtime_state(self, _phase_elapsed) - GatewayRunner._stop_quiesce_and_close_session_dbs(self, timeout, _phase_elapsed) - GatewayRunner._stop_persist_exit_state(self, timed_out, active_agents, _phase_elapsed) - - self._stop_task = asyncio.create_task(_stop_impl()) + self._stop_task = asyncio.create_task(GatewayRunner._stop_impl(self)) await self._stop_task async def wait_for_shutdown(self) -> None: diff --git a/gateway/run_startup.py b/gateway/run_startup.py index bb1b4244c2..d4f8b3fcea 100644 --- a/gateway/run_startup.py +++ b/gateway/run_startup.py @@ -7,10 +7,10 @@ so ``patch("gateway.run.X")`` keeps intercepting them at call time. from __future__ import annotations -import logging -from typing import TYPE_CHECKING import asyncio +import dataclasses import faulthandler +import logging import os import signal import time @@ -19,23 +19,16 @@ from datetime import datetime from gateway.config import Platform from gateway.delivery import looks_like_telegram_private_chat_id from gateway.platforms.base import BasePlatformAdapter, MessageEvent, MessageType +from gateway.session import SessionSource, build_session_key from gateway.restart import ( - DEFAULT_GATEWAY_CRON_DRAIN_TIMEOUT, - GATEWAY_FATAL_CONFIG_EXIT_CODE, - is_global_startup_conflict, + DEFAULT_GATEWAY_CRON_DRAIN_TIMEOUT, GATEWAY_FATAL_CONFIG_EXIT_CODE, is_global_startup_conflict ) from gateway.shutdown_watchdog import ( - DEFAULT_HEARTBEAT_INTERVAL_S, - DEFAULT_LOOP_WATCHDOG_INTERVAL_S, - DEFAULT_LOOP_WATCHDOG_MAX_STRIKES, - DEFAULT_LOOP_WATCHDOG_TIMEOUT_S, - loop_heartbeat_forever, + DEFAULT_HEARTBEAT_INTERVAL_S, DEFAULT_LOOP_WATCHDOG_INTERVAL_S, + DEFAULT_LOOP_WATCHDOG_MAX_STRIKES, DEFAULT_LOOP_WATCHDOG_TIMEOUT_S, loop_heartbeat_forever, ) from typing import Any, Dict, Optional, Tuple -if TYPE_CHECKING: # string annotations only; never imported at runtime (cycle) - from gateway.run import GatewayRunner, TurnRunner # noqa: F401 - # Log-record parity with the origin module. logger = logging.getLogger("gateway.run") @@ -44,15 +37,10 @@ class GatewayStartupMixin: """Startup sequence, resume/restore and handoff methods for GatewayRunner.""" async def _run_startup_resume_event( - self, - adapter: BasePlatformAdapter, - event: MessageEvent, - session_key: str, + self, adapter: BasePlatformAdapter, event: MessageEvent, session_key: str, ) -> None: - """Dispatch one synthetic startup resume and wait for its agent turn. - - Inbound messages stay queued until the resumed turn finishes, else a user message can race it. - """ + """Dispatch one synthetic startup resume and wait for its agent turn (inbound stays queued + until it finishes, else a user message can race it).""" from gateway.run import _AGENT_PENDING_SENTINEL try: await adapter.handle_message(event) @@ -70,25 +58,20 @@ class GatewayStartupMixin: def _queue_startup_restore_event(self, event: MessageEvent) -> None: queue = getattr(self, "_startup_restore_queue", None) if queue is None: - queue = [] - self._startup_restore_queue = queue + queue = self._startup_restore_queue = [] queue.append(event) - try: + with suppress(Exception): source = event.source logger.info( "Queued inbound message during gateway startup restore: platform=%s chat=%s", source.platform.value if source and source.platform else "unknown", source.chat_id if source else "unknown", ) - except Exception: - pass async def _drain_startup_restore_queue(self) -> int: """Replay inbound messages queued while startup auto-resume ran.""" drained = 0 - queue = getattr(self, "_startup_restore_queue", None) - if queue is None: - return 0 + queue = getattr(self, "_startup_restore_queue", None) or [] while queue: event = queue.pop(0) source = getattr(event, "source", None) @@ -108,25 +91,17 @@ class GatewayStartupMixin: return drained def _start_startup_warmup(self) -> None: - """Kick off the boot turn-machinery warm-up in the background. - - Called from ``start()`` right after the startup-restore gate closes so the warm-up overlaps - the network-bound platform connects; ``_finish_startup_restore`` awaits it (bounded). - """ + """Kick off the boot turn-machinery warm-up so it overlaps the network-bound platform + connects; ``_finish_startup_restore`` awaits it (bounded).""" from gateway.run import _startup_warmup_timeout_secs - timeout = _startup_warmup_timeout_secs() - if timeout <= 0: + if _startup_warmup_timeout_secs() <= 0: self._startup_warmup_task = None return - self._startup_warmup_task = asyncio.ensure_future( - self._warm_turn_prerequisites() - ) + self._startup_warmup_task = asyncio.ensure_future(self._warm_turn_prerequisites()) async def _warm_turn_prerequisites(self) -> None: - """Initialize turn machinery on an executor thread before the gate opens. - - Never raises: a failed warm-up degrades to lazy init and must not block startup. - """ + """Initialize turn machinery on an executor thread before the gate opens. Never raises: a + failed warm-up degrades to lazy init and must not block startup.""" from gateway.run import _warm_turn_machinery_sync try: loop = asyncio.get_running_loop() @@ -134,45 +109,52 @@ class GatewayStartupMixin: tool_count = await loop.run_in_executor(None, _warm_turn_machinery_sync) logger.info( "Turn machinery warmed in %.1fs (%d tool schema(s) materialized)", - time.monotonic() - t0, - tool_count, + time.monotonic() - t0, tool_count, ) except Exception: logger.warning( - "Turn-machinery warm-up failed; first inbound turn will " - "initialize lazily", - exc_info=True, + "Turn-machinery warm-up failed; first inbound turn will initialize lazily", exc_info=True, ) async def _await_startup_warmup(self) -> None: - """Bounded wait for the boot warm-up before the inbound gate opens. - - On timeout the gate opens anyway (availability outranks prompt completeness for a WEDGED - init) and the warm-up continues in the background; a late failure is still logged. - """ - from gateway.run import GatewayRunner, _startup_warmup_timeout_secs + """Bounded wait for the boot warm-up. On timeout the gate opens anyway (availability outranks + prompt completeness for a WEDGED init); the warm-up continues and a late failure is logged.""" + from gateway.run import _startup_warmup_timeout_secs task = getattr(self, "_startup_warmup_task", None) if task is None or task.done(): return timeout = _startup_warmup_timeout_secs() if timeout <= 0: return - done, pending = await asyncio.wait({task}, timeout=timeout) + await self._wait_bounded_or_release( + {task}, timeout, + "Turn-machinery warm-up still running after %.0fs; opening inbound gate anyway — the " + "first turn may see lazily initialized machinery (#99373). Warm-up continues in the " + "background.", + "boot turn-machinery warm-up failed after gate release", level=logging.DEBUG, + ) + + async def _wait_bounded_or_release( + self, tasks: set, timeout: float, warn_fmt: str, late_msg: str, *, + level: int = logging.WARNING, track: bool = False, + ) -> set: + """``asyncio.wait`` (which, unlike wait_for/gather+timeout, does NOT cancel on timeout) with + the gate-release warning + late-failure callback every boot gate uses. Returns ``done``. + ``warn_fmt`` takes ``%.0fs`` (timeout) and optionally ``%d`` (pending count).""" + done, pending = await asyncio.wait(tasks, timeout=timeout) if pending: - logger.warning( - "Turn-machinery warm-up still running after %.0fs; opening " - "inbound gate anyway — the first turn may see lazily " - "initialized machinery (#99373). Warm-up continues in the " - "background.", - timeout, - ) - task.add_done_callback( - lambda t: GatewayRunner._log_late_background_failure( - t, - "boot turn-machinery warm-up failed after gate release", - level=logging.DEBUG, - ) - ) + args = (timeout, len(pending)) if warn_fmt.count("%") > 1 else (timeout,) + logger.warning(warn_fmt, *args) + late = self._late_failure_callback(late_msg, level=level) + for task in pending: + task.add_done_callback(late) + if track: + bg = getattr(self, "_background_tasks", None) + if bg is None: + bg = self._background_tasks = set() + bg.add(task) + task.add_done_callback(bg.discard) + return done async def _finish_startup_restore(self) -> None: """Wait (BOUNDED) for startup auto-resume, then release + drain inbound. @@ -187,37 +169,23 @@ class GatewayStartupMixin: if tasks: timeout = _startup_restore_drain_timeout_secs() if timeout > 0: - # asyncio.wait (unlike wait_for / gather+timeout) does NOT cancel pending tasks on - # timeout — the slow resume turn keeps running in the background. - done, pending = await asyncio.wait(tasks, timeout=timeout) - if pending: - logger.warning( - "Startup-restore gate released after %.0fs with %d boot " - "auto-resume turn(s) still running; draining inbound " - "queue now (resume slots already claimed, so no " - "duplicate agents). Slow turn(s) continue in the " - "background.", - timeout, - len(pending), - ) - # These tasks outlive the gate. Their normal done-callback only discards them - # from _background_tasks, so a LATER failure would be silently swallowed. - for task in pending: - task.add_done_callback(self._log_background_resume_result) + # Tasks that outlive the gate get a late-failure callback: their normal done-callback + # only discards them from _background_tasks, so a LATER failure would be swallowed. + done = await self._wait_bounded_or_release( + set(tasks), timeout, + "Startup-restore gate released after %.0fs with %d boot auto-resume turn(s) " + "still running; draining inbound queue now (resume slots already claimed, so no " + "duplicate agents). Slow turn(s) continue in the background.", + "background startup auto-resume task failed after gate release", + level=logging.DEBUG, + ) else: - # Non-positive timeout => opt out of the bound (historical - # "wait forever" behaviour). + # Non-positive timeout => opt out of the bound ("wait forever"). await asyncio.gather(*tasks, return_exceptions=True) done = set(tasks) + report = self._late_failure_callback("startup auto-resume task failed", level=logging.DEBUG) for task in done: - if task.cancelled(): - continue - exc = task.exception() - if exc is not None: - logger.debug( - "startup auto-resume task failed", - exc_info=(type(exc), exc, exc.__traceback__), - ) + report(task) self._startup_restore_tasks = [] # Warm the turn machinery BEFORE the queue drains: replayed (and # fresh) inbound turns must not build skeleton prompts (#99373). @@ -228,45 +196,27 @@ class GatewayStartupMixin: logger.info("Drained %d inbound message(s) queued during startup restore", drained) @staticmethod - def _log_background_resume_result(task: "asyncio.Task") -> None: - """Done-callback for a boot-resume turn that outlived the startup-restore gate.""" - from gateway.run import GatewayRunner - GatewayRunner._log_late_background_failure( - task, - "background startup auto-resume task failed after gate release", - level=logging.DEBUG, - ) + def _late_failure_callback(message: str, *, level: int = logging.WARNING): + """Done-callback for boot-path tasks that outlive the startup-restore gate: surface a late + failure otherwise swallowed once the task leaves ``_background_tasks``. Cancellation is + expected (shutdown), not an error.""" + def _report(task: "asyncio.Task") -> None: + if task.cancelled(): + return + exc = task.exception() + if exc is not None: + logger.log(level, message, exc_info=(type(exc), exc, exc.__traceback__)) + return _report - @staticmethod - def _log_late_background_failure( - task: "asyncio.Task", message: str, *, level: int = logging.WARNING - ) -> None: - """Shared done-callback body for boot-path tasks that outlive the startup-restore gate: - surface a late failure otherwise swallowed once the task leaves ``_background_tasks``. - Cancellation (shutdown) is expected, not an error.""" - if task.cancelled(): - return - exc = task.exception() - if exc is not None: - logger.log( - level, - message, - exc_info=(type(exc), exc, exc.__traceback__), - ) - - async def _await_startup_boot_sends( - self, - *, - planned_restart_notification_pending: bool, - ) -> None: + async def _await_startup_boot_sends(self, *, planned_restart_notification_pending: bool) -> None: """Run boot-path sends without letting them pin the inbound restore gate. - Awaiting ``_send_restart_notification`` / ``_redeliver_pending_obligations`` inline before - the gate releases lets one Telegram flood-control sleep freeze inbound on every platform. - Same bounded ``asyncio.wait`` as the resume gate: on timeout return and let the sends finish - in the background (not cancelled). The ledger claim + ``resume_pending`` clear run INLINE - before the send task exists: bounded DB work, and deferring it let a hung notification - expire the gate with zero rows claimed, so answered turns were replayed AND redelivered. + Awaiting the sends inline before the gate releases lets one Telegram flood-control sleep + freeze inbound on every platform. Same bounded ``asyncio.wait`` as the resume gate: on + timeout return and let the sends finish in the background (not cancelled). The ledger claim + + ``resume_pending`` clear run INLINE before the send task exists (bounded DB work): deferring + it let a hung notification expire the gate with zero rows claimed, so answered turns were + replayed AND redelivered. """ from gateway.run import _clear_planned_restart_notification, _startup_restore_drain_timeout_secs claimed = await self._claim_pending_obligations() @@ -275,9 +225,7 @@ class GatewayStartupMixin: await self._send_restart_notification() if planned_restart_notification_pending: try: - await self._send_home_channel_startup_notifications( - skip_targets=None, - ) + await self._send_home_channel_startup_notifications(skip_targets=None) finally: _clear_planned_restart_notification() await self._redeliver_claimed_obligations(claimed) @@ -285,41 +233,22 @@ class GatewayStartupMixin: boot_task = asyncio.create_task(_boot_sends()) timeout = _startup_restore_drain_timeout_secs() if timeout > 0: - _done, pending = await asyncio.wait({boot_task}, timeout=timeout) - if pending: - logger.warning( - "Boot-path sends still running after %.0fs; releasing " - "inbound gate so other platforms are not frozen. " - "Restart notification / obligation redelivery continue " - "in the background.", - timeout, - ) - boot_task.add_done_callback(self._log_background_boot_send_result) - tasks = getattr(self, "_background_tasks", None) - if tasks is None: - self._background_tasks = set() - tasks = self._background_tasks - tasks.add(boot_task) - boot_task.add_done_callback(tasks.discard) + await self._wait_bounded_or_release( + {boot_task}, timeout, + "Boot-path sends still running after %.0fs; releasing inbound gate so other " + "platforms are not frozen. Restart notification / obligation redelivery continue " + "in the background.", + "background boot-path send failed after gate release: see traceback", track=True, + ) else: await boot_task - @staticmethod - def _log_background_boot_send_result(task: "asyncio.Task") -> None: - """Done-callback for boot-path sends that outlived the restore gate.""" - from gateway.run import GatewayRunner - GatewayRunner._log_late_background_failure( - task, "background boot-path send failed after gate release: see traceback" - ) - async def _clear_resume_pending_for_claimed_obligations( self, claimed: list, *, require_success: bool = False ) -> list: - """Clear resume flags and return rows safe to redeliver. - - Startup recovery stays best-effort. Runtime reconnect recovery is stricter: if the - session-store write fails the response must not be sent, or the turn could be resumed too. - """ + """Clear resume flags and return rows safe to redeliver. Startup recovery is best-effort; + runtime reconnect recovery (``require_success``) is stricter: if the session-store write + fails the response must not be sent, or the turn could be resumed too.""" sendable = [] for row in claimed: session_key = row.get("session_key") or "" @@ -329,14 +258,10 @@ class GatewayStartupMixin: try: await self.async_session_store.clear_resume_pending(session_key) except Exception: - logger.debug( - "clear_resume_pending failed for %s", session_key, - exc_info=True, - ) - if not require_success: - sendable.append(row) - else: - sendable.append(row) + logger.debug("clear_resume_pending failed for %s", session_key, exc_info=True) + if require_success: + continue + sendable.append(row) return sendable async def _claim_pending_obligations(self) -> list: @@ -350,35 +275,24 @@ class GatewayStartupMixin: silent (gateway/delivery_ledger.py). Returns the claimed rows for redelivery. """ try: - from gateway.delivery_ledger import ( - ledger_enabled, - sweep_recoverable, - ) - + from gateway.delivery_ledger import ledger_enabled, sweep_recoverable if not await asyncio.to_thread(ledger_enabled): return [] # Only claim rows whose exact transport owner is connected this boot. A multiplexed # gateway can host several bot identities for one platform; platform-only filtering # would spend a disconnected bot's retry budget merely because another bot is online. _profile_adapters = getattr(self, "_profile_adapters", None) or {} - _deliverable_targets = { - (getattr(p, "value", str(p)), "default") for p in self.adapters - } + _pval = lambda p: getattr(p, "value", str(p)) # noqa: E731 + _deliverable_targets = {(_pval(p), "default") for p in self.adapters} # Legacy rows predate adapter_profile. They are unambiguous only in a non-multiplexed # gateway; fail closed when multiple bot identities share the process. if not _profile_adapters: - _deliverable_targets.update( - (getattr(p, "value", str(p)), None) for p in self.adapters - ) + _deliverable_targets.update((_pval(p), None) for p in self.adapters) for _profile, _adapters in _profile_adapters.items(): - _deliverable_targets.update( - (getattr(p, "value", str(p)), _profile) for p in _adapters - ) - _deliverable = {platform for platform, _ in _deliverable_targets} + _deliverable_targets.update((_pval(p), _profile) for p in _adapters) claimed = await asyncio.to_thread( - sweep_recoverable, - None, - deliverable_platforms=_deliverable, + sweep_recoverable, None, + deliverable_platforms={platform for platform, _ in _deliverable_targets}, deliverable_targets=_deliverable_targets, ) except Exception: @@ -392,79 +306,41 @@ class GatewayStartupMixin: await self._clear_resume_pending_for_claimed_obligations(claimed) return claimed - async def _redeliver_claimed_obligations(self, claimed: list) -> int: - """Redeliver final responses for rows claimed by :meth:`_claim_pending_obligations`. + @staticmethod + async def _release_runtime_claim_quiet(obligation_id, log_fmt: str) -> None: + """Release a runtime delivery-ledger claim as ``send_path_degraded``; log-only on failure.""" + from gateway.delivery_ledger import release_runtime_claim + try: + await asyncio.to_thread(release_runtime_claim, obligation_id, "send_path_degraded") + except Exception: + logger.debug(log_fmt, obligation_id, exc_info=True) - Network half of the split: runs inside the bounded boot-send task, so a flood-limited send - can be abandoned by the restore gate without reopening the turn-replay window. Returns count. - """ + async def _redeliver_claimed_obligations(self, claimed: list) -> int: + """Redeliver final responses for claimed rows (network half of the split): runs inside the + bounded boot-send task, so a flood-limited send can be abandoned by the restore gate without + reopening the turn-replay window. Returns the redelivered count.""" if not claimed: return 0 try: - from gateway.delivery_ledger import ( - RECOVERED_MARKER, - mark_delivered, - mark_failed, - release_runtime_claim, - ) + from gateway.delivery_ledger import RECOVERED_MARKER, mark_delivered, mark_failed except Exception: logger.debug("delivery ledger import failed", exc_info=True) return 0 redelivered = 0 for row in claimed: - try: - platform = Platform(row["platform"]) - except Exception: - logger.debug( - "obligation %s: unknown platform %r", - row["obligation_id"], row.get("platform"), - ) - continue - if "profile" in row: - adapter = self._authorization_adapter( - platform, row.get("profile") - ) - else: - # Startup rows preserve the historical default-adapter route. - adapter = self.adapters.get(platform) + adapter = await self._obligation_adapter(row) if adapter is None: - # Runtime claims have not reached a transport yet. If the - # reconnect vanished before dispatch, release the claim without - # spending an attempt so the next reconnect can retry it. - if row.get("runtime_recovery"): - try: - await asyncio.to_thread( - release_runtime_claim, - row["obligation_id"], - "send_path_degraded", - ) - except Exception: - logger.debug( - "failed to release undispatched runtime obligation %s", - row["obligation_id"], - exc_info=True, - ) - # Startup claims preserve their historical state; attempts cap - # + stale cutoff bound later retries. continue content = row["content"] if row.get("needs_marker"): content = row.get("marker", RECOVERED_MARKER) + content - metadata = ( - {"thread_id": row["thread_id"]} if row.get("thread_id") else None - ) - + metadata = {"thread_id": row["thread_id"]} if row.get("thread_id") else None try: - result = await adapter.send( - chat_id=row["chat_id"], - content=content, - metadata=metadata, - ) + result = await adapter.send(chat_id=row["chat_id"], content=content, metadata=metadata) except Exception as send_err: logger.warning( - "obligation %s: redelivery send raised: %s", - row["obligation_id"], send_err, + "obligation %s: redelivery send raised: %s", row["obligation_id"], send_err ) result = None try: @@ -473,35 +349,48 @@ class GatewayStartupMixin: redelivered += 1 logger.info( "Redelivered recovered final response to %s:%s " - "(obligation %s, attempt %d)", - row["platform"], row["chat_id"], + "(obligation %s, attempt %d)", row["platform"], row["chat_id"], row["obligation_id"], row["attempts"], ) else: await asyncio.to_thread( - mark_failed, - row["obligation_id"], + mark_failed, row["obligation_id"], str(getattr(result, "error", "") or "send failed"), ) except Exception: logger.debug("delivery ledger update failed", exc_info=True) return redelivered + async def _obligation_adapter(self, row: dict): + """Resolve the adapter for a claimed ledger row, or None when it cannot be delivered now.""" + try: + platform = Platform(row["platform"]) + except Exception: + logger.debug( + "obligation %s: unknown platform %r", row["obligation_id"], row.get("platform"), + ) + return None + if "profile" in row: + adapter = self._authorization_adapter(platform, row.get("profile")) + else: + # Startup rows preserve the historical default-adapter route. + adapter = self.adapters.get(platform) + # Runtime claims have not reached a transport yet. If the reconnect vanished before + # dispatch, release the claim without spending an attempt so the next reconnect can retry + # it. Startup claims keep their state; attempts cap + stale cutoff bound later retries. + if adapter is None and row.get("runtime_recovery"): + await self._release_runtime_claim_quiet( + row["obligation_id"], "failed to release undispatched runtime obligation %s" + ) + return adapter + async def _redeliver_pending_obligations(self) -> int: - """Claim + redeliver in one call (:meth:`_claim_pending_obligations` then - :meth:`_redeliver_claimed_obligations`). Stable public shape for tests/external callers; - the startup path calls the halves separately so the DB half runs inline before the - abandonable send task. - """ - return await self._redeliver_claimed_obligations( - await self._claim_pending_obligations() - ) + """Claim + redeliver in one call. Stable shape for tests/external callers; the startup path + calls the halves separately so the DB half runs inline before the abandonable send task.""" + return await self._redeliver_claimed_obligations(await self._claim_pending_obligations()) async def _redeliver_failed_obligations_for_platform( - self, - platform: Platform, - *, - profile: Optional[str] = None, + self, platform: Platform, *, profile: Optional[str] = None, ) -> int: """Replay one adapter identity's transient failures after reconnect. @@ -510,24 +399,13 @@ class GatewayStartupMixin: next restart. Claim/clear/send are best-effort and reuse the startup redelivery contract. """ try: - from gateway.delivery_ledger import ( - ledger_enabled, - release_runtime_claim, - sweep_failed_for_runtime, - ) - + from gateway.delivery_ledger import ledger_enabled, sweep_failed_for_runtime if not await asyncio.to_thread(ledger_enabled): return 0 - claimed = await asyncio.to_thread( - sweep_failed_for_runtime, - platform.value, - profile=profile, - ) + claimed = await asyncio.to_thread(sweep_failed_for_runtime, platform.value, profile=profile) except Exception: logger.debug( - "runtime delivery ledger sweep failed after %s reconnect", - platform.value, - exc_info=True, + "runtime delivery ledger sweep failed after %s reconnect", platform.value, exc_info=True, ) return 0 if not claimed: @@ -540,33 +418,15 @@ class GatewayStartupMixin: ) sendable_ids = {row["obligation_id"] for row in sendable} for row in claimed: - if row["obligation_id"] in sendable_ids: - continue - try: - await asyncio.to_thread( - release_runtime_claim, - row["obligation_id"], - "send_path_degraded", - ) - except Exception: - logger.debug( - "failed to release runtime delivery claim %s", - row["obligation_id"], - exc_info=True, + if row["obligation_id"] not in sendable_ids: + await self._release_runtime_claim_quiet( + row["obligation_id"], "failed to release runtime delivery claim %s" ) return await self._redeliver_claimed_obligations(sendable) - def _schedule_resume_pending_sessions(self, platform=None) -> int: - """Auto-continue fresh restart-interrupted sessions after startup. - - Synthesizes the next turn once adapters are back online; the event text is empty so the - existing ``_is_resume_pending`` injection path owns the recovery wording. Sessions whose - adapter is not in ``self.adapters`` stay ``resume_pending`` for the reconnect watcher, which - re-calls this scoped to that ``platform`` (a reconnecting platform never touches another's - recoveries); sessions with a running agent are skipped so none is resumed twice. - """ - from gateway.run import _AGENT_PENDING_SENTINEL, _auto_continue_freshness_window - window = _auto_continue_freshness_window() + def _resume_pending_candidates(self, platform=None) -> Optional[list]: + """Snapshot resume-pending entries (optionally scoped to ``platform``); None when + enumeration failed or the restart-loop breaker tripped for this boot.""" try: with self.session_store._lock: # noqa: SLF001 — snapshot under lock self.session_store._ensure_loaded_locked() # noqa: SLF001 @@ -580,7 +440,7 @@ class GatewayStartupMixin: ] except Exception as exc: logger.warning("Failed to enumerate resume-pending sessions: %s", exc) - return 0 + return None # Defense-3: break the SIGTERM-respawn loop. Only count this boot when there are restart- # interrupted sessions to resume — a clean boot must not accrue toward the breaker. If too @@ -589,14 +449,44 @@ class GatewayStartupMixin: if candidates: try: from gateway import restart_loop_guard as _rlg - _max_restarts, _window, _max_gap = self._restart_loop_guard_config() - if _rlg.check_and_record( - _max_restarts, _window, max_gap_seconds=_max_gap - ): - return 0 + if _rlg.check_and_record(_max_restarts, _window, max_gap_seconds=_max_gap): + return None except Exception as exc: # noqa: BLE001 — breaker must fail OPEN logger.debug("Restart-loop guard check skipped: %s", exc) + return candidates + + def _resume_owner_authorized(self, session_key: str, source) -> bool: + """Validate the session owner against the CURRENT allowlist: a session created before the + allowlist existed (or whose owner was since removed) must not silently receive a full agent + response just because it carries a resume marker.""" + try: + if self._is_user_authorized(source): + return True + logger.warning( + "Skipping auto-resume for %s: session owner is no " + "longer authorized under the current allowlist", session_key, + ) + except Exception as exc: + logger.warning( + "Skipping auto-resume for %s: authorization check failed: %s", session_key, exc, + ) + return False + + def _schedule_resume_pending_sessions(self, platform=None) -> int: + """Auto-continue fresh restart-interrupted sessions after startup. + + Synthesizes the next turn once adapters are back online; the event text is empty so the + existing ``_is_resume_pending`` injection path owns the recovery wording. Sessions whose + adapter is not in ``self.adapters`` stay ``resume_pending`` for the reconnect watcher, which + re-calls this scoped to that ``platform`` (a reconnecting platform never touches another's + recoveries); sessions with a running agent are skipped so none is resumed twice. + """ + from gateway.run import _AGENT_PENDING_SENTINEL, _auto_continue_freshness_window + window = _auto_continue_freshness_window() + candidates = self._resume_pending_candidates(platform) + if candidates is None: + return 0 now = datetime.now() scheduled = 0 @@ -605,8 +495,7 @@ class GatewayStartupMixin: if marker is not None and (now - marker).total_seconds() > window: continue - # Already being resumed (e.g. scheduled at startup and still - # in-flight) — don't synthesize a second continuation turn. + # Already being resumed (e.g. scheduled at startup, still in-flight) — no second turn. if self._is_session_running(entry.session_key): continue @@ -614,28 +503,11 @@ class GatewayStartupMixin: adapter = self._adapter_for_source(source) if adapter is None: logger.debug( - "Skipping auto-resume for %s: adapter not ready for %s", - entry.session_key, + "Skipping auto-resume for %s: adapter not ready for %s", entry.session_key, getattr(source.platform, "value", source.platform), ) continue - - # Validate the session owner against the current allowlist before auto-resuming: a - # session created before the allowlist existed (or whose owner was since removed) must - # not silently receive a full agent response just because it carries a resume marker. - try: - if not self._is_user_authorized(source): - logger.warning( - "Skipping auto-resume for %s: session owner is no " - "longer authorized under the current allowlist", - entry.session_key, - ) - continue - except Exception as exc: - logger.warning( - "Skipping auto-resume for %s: authorization check failed: %s", - entry.session_key, exc, - ) + if not self._resume_owner_authorized(entry.session_key, source): continue # Claim the session slot *before* spawning the task so an inbound message arriving @@ -648,12 +520,7 @@ class GatewayStartupMixin: # Empty-text internal event: the _is_resume_pending branch in _handle_message_with_agent # prepends the reason-aware system note before the turn runs. - event = MessageEvent( - text="", - message_type=MessageType.TEXT, - source=source, - internal=True, - ) + event = MessageEvent(text="", message_type=MessageType.TEXT, source=source, internal=True) task = asyncio.create_task( self._run_startup_resume_event(adapter, event, entry.session_key) ) @@ -662,55 +529,55 @@ class GatewayStartupMixin: if getattr(self, "_startup_restore_in_progress", False): tasks = getattr(self, "_startup_restore_tasks", None) if tasks is None: - tasks = [] - self._startup_restore_tasks = tasks + tasks = self._startup_restore_tasks = [] tasks.append(task) scheduled += 1 if scheduled: - logger.info( - "Scheduled auto-resume for %d restart-interrupted session(s)", - scheduled, - ) + logger.info("Scheduled auto-resume for %d restart-interrupted session(s)", scheduled) return scheduled def _startup_should_abort(self) -> bool: - return ( - self._restart_requested - or self._draining - or self._shutdown_event.is_set() - ) + return self._restart_requested or self._draining or self._shutdown_event.is_set() + + async def _startup_teardown_adapter(self, adapter, platform) -> None: + """Cancel an adapter's background tasks (best-effort) then disconnect it.""" + try: + await adapter.cancel_background_tasks() + except Exception as e: + logger.debug("✗ %s background-task cancel error: %s", platform.value, e) + await self._safe_adapter_disconnect(adapter, platform) + + def _startup_retry_entry(self, platform, adapter, platform_config, *, queued: bool = True) -> dict: + """Build a ``_failed_platforms`` entry for a platform that failed at startup.""" + return { + "config": platform_config, "attempts": 1, "next_retry": time.monotonic() + 30, + **({"queued_at": time.monotonic()} if queued else {}), + "credential_claim": self._adapter_credential_claim(platform, adapter), + "listener_claim": self._adapter_listener_claim(platform, adapter), + } async def _abort_startup_if_shutdown_requested( - self, - adapter: Optional[BasePlatformAdapter] = None, - platform: Optional[Platform] = None, + self, adapter: Optional[BasePlatformAdapter] = None, platform: Optional[Platform] = None ) -> bool: """Clean up and exit startup when restart/shutdown begins mid-startup.""" if not self._startup_should_abort(): return False if adapter is not None and platform is not None: - try: - await adapter.cancel_background_tasks() - except Exception as e: - logger.debug("✗ %s background-task cancel error: %s", platform.value, e) - await self._safe_adapter_disconnect(adapter, platform) + await self._startup_teardown_adapter(adapter, platform) stop_task = self._stop_task current_task = asyncio.current_task() if stop_task is not None and stop_task is not current_task: await stop_task elif not self._shutdown_event.is_set(): await self.stop( - restart=self._restart_requested, - detached_restart=self._restart_detached, + restart=self._restart_requested, detached_restart=self._restart_detached, service_restart=self._restart_via_service, ) return True def _start_loop_liveness_guards(self, loop: asyncio.AbstractEventLoop) -> None: - """Arm the selector floor and out-of-loop watchdog before adapters. - - Disabled entirely with ``gateway.loop_watchdog: false`` in config.yaml (config-only knob). - """ + """Arm the selector floor and out-of-loop watchdog before adapters. Disabled entirely with + ``gateway.loop_watchdog: false`` in config.yaml (config-only knob).""" from gateway.run import _arm_loop_floor_timer, start_loop_liveness_watchdog config = getattr(self, "config", None) if config is not None and not getattr(config, "loop_watchdog", True): @@ -726,64 +593,40 @@ class GatewayStartupMixin: try: # getattr defaults cover the config=None / bare-object test path; config-loaded # values are already validated+clamped by GatewayConfig.from_dict; no re-clamping. - interval = getattr( - config, - "loop_watchdog_probe_interval_s", - DEFAULT_LOOP_WATCHDOG_INTERVAL_S, - ) - timeout = getattr( - config, - "loop_watchdog_probe_timeout_s", - DEFAULT_LOOP_WATCHDOG_TIMEOUT_S, - ) - strikes = getattr( - config, - "loop_watchdog_max_strikes", - DEFAULT_LOOP_WATCHDOG_MAX_STRIKES, - ) self._loop_liveness_watchdog = start_loop_liveness_watchdog( loop, - probe_interval=float(interval), - probe_timeout=float(timeout), - max_strikes=int(strikes), + probe_interval=float(getattr( + config, "loop_watchdog_probe_interval_s", DEFAULT_LOOP_WATCHDOG_INTERVAL_S + )), + probe_timeout=float(getattr( + config, "loop_watchdog_probe_timeout_s", DEFAULT_LOOP_WATCHDOG_TIMEOUT_S + )), + max_strikes=int(getattr( + config, "loop_watchdog_max_strikes", DEFAULT_LOOP_WATCHDOG_MAX_STRIKES + )), ) except Exception: logger.debug("Failed to start gateway loop liveness watchdog", exc_info=True) def _stop_loop_liveness_guards(self) -> None: - """Disarm lifetime liveness guards before shutdown can load the loop.""" - watchdog = getattr(self, "_loop_liveness_watchdog", None) - self._loop_liveness_watchdog = None - if watchdog is not None: - try: - watchdog.stop() - except Exception: - logger.debug("Failed to stop gateway loop liveness watchdog", exc_info=True) - - floor_timer = getattr(self, "_loop_floor_timer_handle", None) - self._loop_floor_timer_handle = None - if floor_timer is not None: - try: - floor_timer.cancel() - except Exception: - logger.debug("Failed to cancel gateway loop floor timer", exc_info=True) - - # Also disarm the heartbeat writer task: once shutdown starts loading the loop, a heartbeat - # that keeps refreshing the file makes a draining gateway look healthy to external probes. - heartbeat = getattr(self, "_loop_heartbeat_task", None) - self._loop_heartbeat_task = None - if heartbeat is not None: - try: - heartbeat.cancel() - except Exception: - logger.debug("Failed to cancel gateway loop heartbeat task", exc_info=True) + """Disarm lifetime liveness guards before shutdown can load the loop — including the + heartbeat writer, which would otherwise make a draining gateway look healthy to probes.""" + for attr, method, what in ( + ("_loop_liveness_watchdog", "stop", "stop gateway loop liveness watchdog"), + ("_loop_floor_timer_handle", "cancel", "cancel gateway loop floor timer"), + ("_loop_heartbeat_task", "cancel", "cancel gateway loop heartbeat task"), + ): + guard = getattr(self, attr, None) + setattr(self, attr, None) + if guard is not None: + try: + getattr(guard, method)() + except Exception: + logger.debug("Failed to %s", what, exc_info=True) async def _consume_clean_shutdown_marker(self, marker_path) -> int: - """Discard orphan turn markers before consuming a clean-exit receipt. - - If persistence or marker removal fails, startup must fail closed: continuing with the old - receipt would let a later unclean exit masquerade as clean and discard interrupted turns. - """ + """Discard orphan turn markers before consuming a clean-exit receipt. Raises (fail closed): + continuing with the old receipt would let a later unclean exit masquerade as clean.""" discarded = await self.async_session_store.discard_active_turn_markers() marker_path.unlink() return discarded @@ -795,16 +638,13 @@ class GatewayStartupMixin: fallback = 0 try: agent_timeout = max(1.0, _float_env("HERMES_AGENT_TIMEOUT", 1800)) - marker_max_age = max(60 * 60, int(agent_timeout * 2)) exact = await self.async_session_store.recover_interrupted_turns( - max_age_seconds=marker_max_age + max_age_seconds=max(60 * 60, int(agent_timeout * 2)) ) except Exception as exc: logger.warning("Exact active-turn recovery on startup failed: %s", exc) try: - fallback = await self.async_session_store.suspend_recently_active( - max_age_seconds=120 - ) + fallback = await self.async_session_store.suspend_recently_active(max_age_seconds=120) except Exception as exc: logger.warning("Legacy session recovery on startup failed: %s", exc) return exact, fallback @@ -812,7 +652,6 @@ class GatewayStartupMixin: @staticmethod def _start_hosted_room_worker_sync(): """Start the local Group Chat worker without importing the dashboard.""" - import tui_gateway.server # noqa: F401 from tui_gateway import methods_groups @@ -831,32 +670,23 @@ class GatewayStartupMixin: async def _hosted_room_worker_watcher(self, interval: float = 1.0) -> None: """Keep the room worker alive for the messaging gateway lifetime.""" - while self._running: await self._ensure_hosted_room_worker() await asyncio.sleep(interval) async def _stop_hosted_room_worker(self, timeout: float = 5.0) -> bool: """Pause room execution durably without interrupting accepted turns.""" - from tui_gateway import methods_groups - - return await asyncio.to_thread( - methods_groups.stop_hosted_room_service, - timeout=timeout, - ) + return await asyncio.to_thread(methods_groups.stop_hosted_room_service, timeout=timeout) def _start_loop_heartbeat_task(self) -> None: - """Start the loop-liveness heartbeat task, idempotent. - - An asyncio task so a frozen loop stops refreshing ``state/gateway.heartbeat``; cancelled - with the other background tasks in stop(). Best-effort — must never abort startup. - """ + """Start the loop-liveness heartbeat task (idempotent, best-effort). An asyncio task so a + frozen loop stops refreshing ``state/gateway.heartbeat``; cancelled with the others in stop().""" try: _existing_hb = getattr(self, "_loop_heartbeat_task", None) if _existing_hb is not None and not _existing_hb.done(): return - self._loop_heartbeat_task = asyncio.create_task( + task = self._loop_heartbeat_task = asyncio.create_task( loop_heartbeat_forever( interval_s=DEFAULT_HEARTBEAT_INTERVAL_S, start_time=getattr(self, "_gateway_started_at", 0.0), @@ -865,52 +695,42 @@ class GatewayStartupMixin: # PERMANENT for the process lifetime, same as a _spawn_supervised watcher — tag it so # _scale_to_zero_has_live_background_work() doesn't treat an armed, otherwise-idle # gateway as busy forever. - self._loop_heartbeat_task._hermes_supervised_watcher = True # type: ignore[attr-defined] + task._hermes_supervised_watcher = True # type: ignore[attr-defined] _bg = getattr(self, "_background_tasks", None) if _bg is not None: - _bg.add(self._loop_heartbeat_task) - self._loop_heartbeat_task.add_done_callback(_bg.discard) + _bg.add(task) + task.add_done_callback(_bg.discard) except Exception: logger.debug("Failed to start gateway loop heartbeat", exc_info=True) + def _open_faulthandler_log(self): + """Open (append) ``/gateway_faulthandler.log``, creating the directory.""" + from gateway.run import get_hermes_home + log_dir = getattr(self.config, "log_dir", None) or os.path.join( + str(get_hermes_home()), "logs", + ) + os.makedirs(log_dir, exist_ok=True) + return open(os.path.join(log_dir, "gateway_faulthandler.log"), "a", encoding="utf-8") + def _start_install_faulthandler(self) -> None: """Enable faulthandler (stderr or a log file) plus the SIGUSR2 stack-dump hook.""" - from gateway.run import get_hermes_home - # Enable faulthandler for stack dumps on freezes/crashes. Falls back to a log file when - # sys.stderr is None (Windows VBS / pythonw / detached service) — otherwise the gateway - # would die here and take every adapter offline. + # Falls back to a log file when sys.stderr is None (Windows VBS / pythonw / detached + # service) — otherwise the gateway would die here and take every adapter offline. try: faulthandler.enable() except (RuntimeError, ValueError, OSError): try: - _fh_log_dir = getattr(self.config, "log_dir", None) or os.path.join( - str(get_hermes_home()), - "logs", - ) - os.makedirs(_fh_log_dir, exist_ok=True) - _fh_enable_path = os.path.join(_fh_log_dir, "gateway_faulthandler.log") - _fh_enable_file = open(_fh_enable_path, "a", encoding="utf-8") - faulthandler.enable(file=_fh_enable_file, all_threads=True) + faulthandler.enable(file=self._open_faulthandler_log(), all_threads=True) except Exception: logger.debug("faulthandler.enable() unavailable", exc_info=True) - # Also dump stacks to a rotating file for off-line analysis under a service manager that - # doesn't capture stderr. faulthandler.register()/SIGUSR2 are POSIX-only: skip the signal- - # triggered file dump on Windows (faulthandler.enable() above still covers fatal errors). + # Also dump stacks to a file on SIGUSR2 for off-line analysis under a service manager that + # doesn't capture stderr. faulthandler.register()/SIGUSR2 are POSIX-only: skip on Windows + # (faulthandler.enable() above still covers fatal errors). _sigusr2 = getattr(signal, "SIGUSR2", None) if _sigusr2 is not None and hasattr(faulthandler, "register"): try: - _log_dir = getattr(self.config, "log_dir", None) or os.path.join( - str(get_hermes_home()), - "logs", - ) - _faulthandler_path = os.path.join(_log_dir, "gateway_faulthandler.log") - os.makedirs(_log_dir, exist_ok=True) - _fh = open(_faulthandler_path, "a", encoding="utf-8") faulthandler.register( - _sigusr2, - file=_fh, - all_threads=True, - chain=True, + _sigusr2, file=self._open_faulthandler_log(), all_threads=True, chain=True, ) except Exception: logger.debug("Could not set up faulthandler file logging", exc_info=True) @@ -929,15 +749,69 @@ class GatewayStartupMixin: # if the loop isn't live, startup has NOT reached the milestone and it must stay armed. try: from gateway.startup_watchdog import disarm_startup_watchdog - disarm_startup_watchdog() except Exception: logger.debug("Startup watchdog disarm failed", exc_info=True) logger.info("Session storage: %s", self.config.sessions_dir) + self._start_log_systemd_timing_alignment() + # Log the resolved max_iterations budget so operators can verify the config.yaml → env + # bridge at a glance (instead of silently running at a stale .env value for weeks). + with suppress(Exception): + logger.info( + "Agent budget: max_iterations=%d (agent.max_turns from config.yaml, " + "or HERMES_MAX_ITERATIONS from .env, or default 500)", + int(os.getenv("HERMES_MAX_ITERATIONS", "500")), + ) + # Redaction is ON by default; warn prominently when an operator has explicitly opted out so + # the downgrade isn't forgotten. The redactor snapshots its state at import time, so this + # log line is the source of truth for the process lifetime. + with suppress(Exception): + _redact_raw = os.getenv("HERMES_REDACT_SECRETS", "true") + if _redact_raw.lower() in {"1", "true", "yes", "on"}: + logger.info( + "Secret redaction: ENABLED (tool output, logs, and chat " + "responses are scrubbed before delivery)" + ) + else: + logger.warning( + "Secret redaction: DISABLED (HERMES_REDACT_SECRETS=%s). " + "API keys and tokens may appear verbatim in chat output, " + "session JSONs, and logs. Set security.redact_secrets: true " + "in config.yaml to re-enable.", _redact_raw, + ) + with suppress(Exception): + from hermes_cli.profiles import get_active_profile_name + _profile = get_active_profile_name() + if _profile and _profile != "default": + logger.info("Active profile: %s", _profile) + with suppress(Exception): + from gateway.status import write_runtime_status + write_runtime_status( + gateway_state="starting", exit_reason=None, clear_profile_platforms=True + ) + try: + from hermes_cli.config import load_config + from agent.monitoring.gateway_health_export import start_gateway_health_export + self._gateway_health_export_runtime = start_gateway_health_export(load_config()) + if getattr(self._gateway_health_export_runtime, "enabled", False): + logger.info("Gateway health OTLP export: enabled") + except Exception: + logger.debug("gateway health OTLP export startup failed", exc_info=True) - # Sanity-check that systemd's TimeoutStopSec covers our drain window: a unit file from - # before a hermes-agent upgrade (no ``hermes setup`` re-run) may encode the old default, - # so SIGKILL hits mid-drain and looks like a phantom kill in the journal. Never raises. + # Log any active supply-chain security advisories. Deliberately does NOT block startup or + # surface inline to users — only the operator can act (uninstall, rotate credentials). + try: + from hermes_cli.security_advisories import detect_compromised, gateway_log_message + _adv_msg = gateway_log_message(detect_compromised()) + if _adv_msg: + logger.warning("%s", _adv_msg) + logger.warning("Run `hermes doctor` on the gateway host for full remediation steps.") + except Exception: + logger.debug("security advisory check failed at gateway startup", exc_info=True) + + def _start_log_systemd_timing_alignment(self) -> None: + """Warn when systemd's TimeoutStopSec does not cover the drain window (a unit file from before + an upgrade may encode the old default, so SIGKILL hits mid-drain). Never raises.""" try: from gateway.shutdown_forensics import check_systemd_timing_alignment _alignment = check_systemd_timing_alignment( @@ -951,159 +825,44 @@ class GatewayStartupMixin: "systemd may SIGKILL the gateway mid-drain. Run " "`hermes gateway install --force` to regenerate the unit, or " "shorten agent.restart_drain_timeout / agent.cron_drain_timeout.", - _alignment.get("unit", "(unknown)"), - _alignment["timeout_stop_sec"], + _alignment.get("unit", "(unknown)"), _alignment["timeout_stop_sec"], _alignment["drain_timeout"], - _alignment.get( - "cron_drain_timeout", DEFAULT_GATEWAY_CRON_DRAIN_TIMEOUT - ), + _alignment.get("cron_drain_timeout", DEFAULT_GATEWAY_CRON_DRAIN_TIMEOUT), _alignment["expected_min"], ) except Exception as _e: logger.debug("check_systemd_timing_alignment failed: %s", _e) - # Log the resolved max_iterations budget so operators can verify the config.yaml → env - # bridge at a glance (instead of silently running at a stale .env value for weeks). - try: - _effective_max_iter = int(os.getenv("HERMES_MAX_ITERATIONS", "500")) - logger.info( - "Agent budget: max_iterations=%d (agent.max_turns from config.yaml, " - "or HERMES_MAX_ITERATIONS from .env, or default 500)", - _effective_max_iter, - ) - except Exception: - pass - # Redaction is ON by default; warn prominently when an operator has explicitly opted out so - # the downgrade isn't forgotten. The redactor snapshots its state at import time, so this - # log line is the source of truth for the process lifetime. - try: - _redact_raw = os.getenv("HERMES_REDACT_SECRETS", "true") - _redact_on = _redact_raw.lower() in {"1", "true", "yes", "on"} - if _redact_on: - logger.info( - "Secret redaction: ENABLED (tool output, logs, and chat " - "responses are scrubbed before delivery)" - ) - else: - logger.warning( - "Secret redaction: DISABLED (HERMES_REDACT_SECRETS=%s). " - "API keys and tokens may appear verbatim in chat output, " - "session JSONs, and logs. Set security.redact_secrets: true " - "in config.yaml to re-enable.", - _redact_raw, - ) - except Exception: - pass - try: - from hermes_cli.profiles import get_active_profile_name - _profile = get_active_profile_name() - if _profile and _profile != "default": - logger.info("Active profile: %s", _profile) - except Exception: - pass - try: - from gateway.status import write_runtime_status - write_runtime_status( - gateway_state="starting", - exit_reason=None, - clear_profile_platforms=True, - ) - except Exception: - pass - try: - from hermes_cli.config import load_config - from agent.monitoring.gateway_health_export import start_gateway_health_export - self._gateway_health_export_runtime = start_gateway_health_export(load_config()) - if getattr(self._gateway_health_export_runtime, "enabled", False): - logger.info("Gateway health OTLP export: enabled") - except Exception: - logger.debug("gateway health OTLP export startup failed", exc_info=True) - # Log any active supply-chain security advisories. Deliberately does NOT block startup or - # surface inline to users — only the operator can act (uninstall, rotate credentials). - try: - from hermes_cli.security_advisories import ( - detect_compromised, - gateway_log_message, - ) - _adv_hits = detect_compromised() - _adv_msg = gateway_log_message(_adv_hits) - if _adv_msg: - logger.warning("%s", _adv_msg) - logger.warning( - "Run `hermes doctor` on the gateway host for full " - "remediation steps." - ) - except Exception: - logger.debug( - "security advisory check failed at gateway startup", - exc_info=True, - ) + # Builtin platforms whose ``

_ALLOWED_USERS`` / ``

_ALLOW_ALL_USERS`` env vars count as + # "an allowlist is configured" / "open access opted in"; plugin platforms are appended at + # check time. + _ALLOWLIST_ENV_PLATFORMS = ( + "TELEGRAM", "DISCORD", "WHATSAPP", "WHATSAPP_CLOUD", "SLACK", "SIGNAL", "EMAIL", "SMS", + "MATTERMOST", "MATRIX", "DINGTALK", "FEISHU", "WECOM", "WECOM_CALLBACK", "WEIXIN", + "BLUEBUBBLES", "QQ", "YUANBAO", + ) + _BUILTIN_ALLOWED_USERS_VARS = tuple(f"{p}_ALLOWED_USERS" for p in _ALLOWLIST_ENV_PLATFORMS) + ( + "SIGNAL_GROUP_ALLOWED_USERS", "TELEGRAM_GROUP_ALLOWED_USERS", + "TELEGRAM_GROUP_ALLOWED_CHATS", "GATEWAY_ALLOWED_USERS", + ) + _BUILTIN_ALLOW_ALL_VARS = tuple(f"{p}_ALLOW_ALL_USERS" for p in _ALLOWLIST_ENV_PLATFORMS) def _start_check_access_policy(self) -> bool: """Warn about missing allowlists; return True when startup must be refused.""" from gateway.run import ( - _OWN_POLICY_OPEN_ENV, - _own_policy_open_startup_violation, - _write_runtime_status_quiet, + _OWN_POLICY_OPEN_ENV, _own_policy_open_startup_violation, _write_runtime_status_quiet ) - # Warn if no user allowlists are configured and open access is not opted in - _builtin_allowed_vars = ( - "TELEGRAM_ALLOWED_USERS", "DISCORD_ALLOWED_USERS", - "WHATSAPP_ALLOWED_USERS", "WHATSAPP_CLOUD_ALLOWED_USERS", - "SLACK_ALLOWED_USERS", - "SIGNAL_ALLOWED_USERS", "SIGNAL_GROUP_ALLOWED_USERS", - "TELEGRAM_GROUP_ALLOWED_USERS", - "TELEGRAM_GROUP_ALLOWED_CHATS", - "EMAIL_ALLOWED_USERS", - "SMS_ALLOWED_USERS", "MATTERMOST_ALLOWED_USERS", - "MATRIX_ALLOWED_USERS", "DINGTALK_ALLOWED_USERS", - "FEISHU_ALLOWED_USERS", - "WECOM_ALLOWED_USERS", - "WECOM_CALLBACK_ALLOWED_USERS", - "WEIXIN_ALLOWED_USERS", - "BLUEBUBBLES_ALLOWED_USERS", - "QQ_ALLOWED_USERS", - "YUANBAO_ALLOWED_USERS", - "GATEWAY_ALLOWED_USERS", - ) - _builtin_allow_all_vars = ( - "TELEGRAM_ALLOW_ALL_USERS", "DISCORD_ALLOW_ALL_USERS", - "WHATSAPP_ALLOW_ALL_USERS", "WHATSAPP_CLOUD_ALLOW_ALL_USERS", - "SLACK_ALLOW_ALL_USERS", - "SIGNAL_ALLOW_ALL_USERS", "EMAIL_ALLOW_ALL_USERS", - "SMS_ALLOW_ALL_USERS", "MATTERMOST_ALLOW_ALL_USERS", - "MATRIX_ALLOW_ALL_USERS", "DINGTALK_ALLOW_ALL_USERS", - "FEISHU_ALLOW_ALL_USERS", - "WECOM_ALLOW_ALL_USERS", - "WECOM_CALLBACK_ALLOW_ALL_USERS", - "WEIXIN_ALLOW_ALL_USERS", - "BLUEBUBBLES_ALLOW_ALL_USERS", - "QQ_ALLOW_ALL_USERS", - "YUANBAO_ALLOW_ALL_USERS", - ) - # Also pick up plugin-registered platforms — each entry can declare its own - # allowed_users_env / allow_all_env, so the warning stays accurate as plugins (IRC) arrive. - _plugin_allowed_vars: tuple = () - _plugin_allow_all_vars: tuple = () - try: + # Plugin-registered platforms declare their own allowed_users_env / allow_all_env, so the + # warning stays accurate as plugins (IRC) arrive. + allowed_vars = list(self._BUILTIN_ALLOWED_USERS_VARS) + allow_all_vars = ["GATEWAY_ALLOW_ALL_USERS", *self._BUILTIN_ALLOW_ALL_VARS] + with suppress(Exception): from gateway.platform_registry import platform_registry - _plugin_allowed_vars = tuple( - e.allowed_users_env for e in platform_registry.plugin_entries() - if e.allowed_users_env - ) - _plugin_allow_all_vars = tuple( - e.allow_all_env for e in platform_registry.plugin_entries() - if e.allow_all_env - ) - except Exception: - pass - _any_allowlist = any( - os.getenv(v) for v in _builtin_allowed_vars + _plugin_allowed_vars - ) - _allow_all = os.getenv("GATEWAY_ALLOW_ALL_USERS", "").lower() in {"true", "1", "yes"} or any( - os.getenv(v, "").lower() in {"true", "1", "yes"} - for v in _builtin_allow_all_vars + _plugin_allow_all_vars - ) + entries = platform_registry.plugin_entries() + allowed_vars += [e.allowed_users_env for e in entries if e.allowed_users_env] + allow_all_vars += [e.allow_all_env for e in entries if e.allow_all_env] + _any_allowlist = any(os.getenv(v) for v in allowed_vars) + _allow_all = any(os.getenv(v, "").lower() in {"true", "1", "yes"} for v in allow_all_vars) if not _any_allowlist and not _allow_all: logger.warning( "No env user allowlists configured. Messaging platforms default to " @@ -1116,15 +875,12 @@ class GatewayStartupMixin: reason = _own_policy_open_startup_violation(self.config) if reason: platform_value = reason.split(":", 1)[0] - allow_all_env = None - for platform, open_env in _OWN_POLICY_OPEN_ENV.items(): - if platform.value == platform_value: - allow_all_env = open_env[2] - break + allow_all_env = next( + (env[2] for p, env in _OWN_POLICY_OPEN_ENV.items() if p.value == platform_value), None + ) logger.error( "Refusing to start: %s has dm_policy/group_policy set to 'open' " - "but neither GATEWAY_ALLOW_ALL_USERS nor %s is enabled.", - platform_value, + "but neither GATEWAY_ALLOW_ALL_USERS nor %s is enabled.", platform_value, allow_all_env or "a platform allow-all flag", ) _write_runtime_status_quiet(gateway_state="startup_failed", exit_reason=reason) @@ -1132,9 +888,9 @@ class GatewayStartupMixin: return True return False - async def _start_recover_previous_run(self) -> None: - """Plugins, relay, hooks, then crash/clean-exit recovery of processes and sessions.""" - from gateway.run import _hermes_home + @staticmethod + def _start_register_plugins_relay_hooks() -> None: + """Plugin discovery, relay registration and shell-hook/webhook registration. Never raises.""" # Discover Python plugins before shell hooks so plugin block decisions take precedence in # tie cases. Explicit here because the gateway lazily imports run_agent per request, so # the discover_plugins() side-effect in model_tools.py is NOT guaranteed to have run yet. @@ -1142,55 +898,42 @@ class GatewayStartupMixin: from hermes_cli.plugins import discover_plugins discover_plugins() except Exception: - logger.warning( - "plugin discovery failed at gateway startup", exc_info=True, - ) + logger.warning("plugin discovery failed at gateway startup", exc_info=True) - # Register the generic relay adapter only if GATEWAY_RELAY_URL / gateway.relay_url is set. - # No URL -> no-op, so direct/single-tenant deployments are unaffected. + # Generic relay adapter only if GATEWAY_RELAY_URL / gateway.relay_url is set; no URL -> no-op. try: from gateway.relay import ( - register_relay_adapter, - relay_url, - self_provision_relay, - send_relay_policy, + register_relay_adapter, relay_url, self_provision_relay, send_relay_policy ) - # Boot-time relay self-provision: resolve the agent's NAS token -> POST /relay/provision # -> set GATEWAY_RELAY_* in os.environ BEFORE registration reads them. Never raises. self_provision_relay() - if register_relay_adapter(): logger.info("relay adapter registered (connector at %s)", relay_url()) # Declare this gateway's relevance policy (mention-gating / free-response / allow- - # bots) to the connector so the SAME behavior governs relay delivery (Phase 6 Unit - # ζ). Runs after the secret is resolved; never raises, never blocks boot. + # bots) to the connector so the SAME behavior governs relay delivery. Runs after + # the secret is resolved; never raises, never blocks boot. send_relay_policy() except Exception: - logger.warning( - "relay adapter registration failed at gateway startup", exc_info=True, - ) + logger.warning("relay adapter registration failed at gateway startup", exc_info=True) - # Register declarative shell hooks from cli-config.yaml. Gateway has no TTY, so consent must - # come from --accept-hooks, HERMES_ACCEPT_HOOKS, or hooks_auto_accept: true; pass - # accept_hooks=False and let register_from_config resolve env + config. Never blocks startup. + # Declarative shell hooks from cli-config.yaml. Gateway has no TTY, so consent must come + # from --accept-hooks, HERMES_ACCEPT_HOOKS, or hooks_auto_accept: true; pass + # accept_hooks=False and let register_from_config resolve env + config. try: from hermes_cli.config import load_config from agent.shell_hooks import register_from_config _hooks_cfg = load_config() register_from_config(_hooks_cfg, accept_hooks=False) - - from agent.outbound_webhooks import ( - register_from_config as register_outbound_webhooks, - ) + from agent.outbound_webhooks import register_from_config as register_outbound_webhooks register_outbound_webhooks(_hooks_cfg) except Exception: - logger.debug( - "shell-hook registration failed at gateway startup", - exc_info=True, - ) + logger.debug("shell-hook registration failed at gateway startup", exc_info=True) - # Discover and load event hooks + async def _start_recover_previous_run(self) -> None: + """Plugins, relay, hooks, then crash/clean-exit recovery of processes and sessions.""" + from gateway.run import _hermes_home + self._start_register_plugins_relay_hooks() self.hooks.discover_and_load() # Recover background processes from checkpoint (crash recovery) @@ -1213,25 +956,17 @@ class GatewayStartupMixin: except Exception as exc: logger.error( "Clean-start marker cleanup failed; refusing startup so the " - "clean-exit receipt cannot mask a later unclean exit: %s", - exc, + "clean-exit receipt cannot mask a later unclean exit: %s", exc, ) raise RuntimeError("clean-start recovery cleanup failed") from exc if discarded: - logger.info( - "Discarded %d orphan active-turn marker(s) after clean shutdown", - discarded, - ) + logger.info("Discarded %d orphan active-turn marker(s) after clean shutdown", discarded) else: exact, fallback = await self._recover_unclean_sessions() - recovered = exact + fallback - if recovered: + if exact + fallback: logger.info( "Marked %d in-flight session(s) as resumable from previous run " - "(%d exact, %d legacy)", - recovered, - exact, - fallback, + "(%d exact, %d legacy)", exact + fallback, exact, fallback, ) # Stuck-loop detection: a session active across 3+ consecutive restarts is probably looping @@ -1244,18 +979,13 @@ class GatewayStartupMixin: logger.debug("Stuck-loop detection failed: %s", e) async def _start_prefilter_platforms(self) -> Tuple[bool, int, list, list]: - """Create + wire an adapter per enabled platform (serial pre-filter, no connects). - - Returns (aborted, enabled_platform_count, multiplex_skipped_platforms, pending_connects). - """ + """Create + wire an adapter per enabled platform (no connects). Returns + (aborted, enabled_platform_count, multiplex_skipped_platforms, pending_connects).""" from gateway.run import _platform_has_bot_credential enabled_platform_count = 0 _multiplex_on = bool(getattr(self.config, "multiplex_profiles", False)) _multiplex_skipped_platforms: list[Platform] = [] - # Initialize and connect each configured platform. connect() calls run concurrently so one - # slow/failing platform (e.g. Telegram behind a dead proxy) cannot delay the others by a - # full timeout window; the cheap serial pre-filter and per-platform timeouts are unchanged. - _pending_connects = [] # (platform, platform_config, adapter) + _pending_connects = [] # (platform, platform_config, adapter); connected concurrently later for platform, platform_config in self.config.platforms.items(): if await self._abort_startup_if_shutdown_requested(): return True, enabled_platform_count, _multiplex_skipped_platforms, _pending_connects @@ -1269,8 +999,7 @@ class GatewayStartupMixin: logger.info( "Skipping %s on default profile: no bot credential in this " "profile's secrets. Secondary multiplexed profiles that " - "provide the token will still connect.", - platform.value, + "provide the token will still connect.", platform.value, ) _multiplex_skipped_platforms.append(platform) continue @@ -1279,41 +1008,25 @@ class GatewayStartupMixin: adapter = self._create_adapter(platform, platform_config) if not adapter: # Distinguish between missing builtin deps and missing plugin - _pval = platform.value - _builtin_names = {m.value for m in Platform.__members__.values()} - if _pval not in _builtin_names: + if platform.value in {m.value for m in Platform.__members__.values()}: + logger.warning("No adapter available for %s", platform.value) + else: logger.warning( "No adapter for '%s' -- is the plugin installed? " "(platform is enabled in config.yaml but no plugin registered it)", - _pval, + platform.value, ) - else: - logger.warning("No adapter available for %s", _pval) continue - - # Set up message + fatal error handlers. Under multiplexing the default profile needs - # the same whole-handler runtime scope as a secondary profile: authorization and prompt - # rendering both run before the narrower agent-turn scope is installed. - adapter.set_message_handler(self._primary_message_handler()) - adapter.set_fatal_error_handler(self._handle_adapter_fatal_error) - adapter.set_session_store(self.session_store) - adapter.set_busy_session_handler(self._handle_active_session_busy_message) - _set_reaction = getattr(adapter, "set_reaction_handler", None) - if callable(_set_reaction): - _set_reaction(self._handle_reaction_event) - adapter.set_topic_recovery_fn(self._recover_telegram_topic_thread_id) - adapter.set_authorization_check(self._make_adapter_auth_check(adapter.platform)) - adapter.set_platform_event_handler(self._primary_platform_event_handler()) - adapter._busy_text_mode = self._busy_text_mode + # Under multiplexing the default profile needs the same whole-handler runtime scope as + # a secondary profile: authorization and prompt rendering both run before the narrower + # agent-turn scope is installed. + self._wire_adapter_handlers(adapter) _pending_connects.append((platform, platform_config, adapter)) return False, enabled_platform_count, _multiplex_skipped_platforms, _pending_connects async def _start_connect_pending(self, _pending_connects: list) -> Optional[list]: - """Connect the pre-filtered adapters concurrently. - - Returns the raw per-platform results, or None when a restart/shutdown aborted startup - mid-connect (adapters already torn down). - """ + """Connect the pre-filtered adapters concurrently. Returns the raw per-platform results, or + None when a restart/shutdown aborted startup mid-connect (adapters already torn down).""" async def _connect_one_startup(p, p_cfg, adp): """Connect a single platform; never let one block the others (#83791).""" if await self._abort_startup_if_shutdown_requested(adp, p): @@ -1328,77 +1041,59 @@ class GatewayStartupMixin: return (p, adp, p_cfg, "exception", _exc) return (p, adp, p_cfg, "ok" if ok else "failed", None) - if _pending_connects: - # Abort-aware concurrent wait (parity with the serial loop's between-platforms check): a - # restart/shutdown requested mid-connect must cancel still-pending connects, clean up the - # ones already completed, and abort startup. - _task_map: dict = {} - for (p, c, a) in _pending_connects: - _t = asyncio.ensure_future(_connect_one_startup(p, c, a)) - _task_map[_t] = (p, c, a) - _pending_tasks = set(_task_map) - _abort_mid_connect = False - while _pending_tasks: - _done, _pending_tasks = await asyncio.wait( - _pending_tasks, timeout=0.05 - ) - if _pending_tasks and self._startup_should_abort(): - _abort_mid_connect = True - break - if _abort_mid_connect: - # Cancel and fully settle the in-flight connects FIRST, so a completed adapter's - # disconnect cannot unblock a sibling's connect() before the sibling is cancelled. - for _t in _pending_tasks: - _t.cancel() - await asyncio.gather(*_pending_tasks, return_exceptions=True) - for _t in _pending_tasks: - _p, _c, _a = _task_map[_t] - try: - await _a.cancel_background_tasks() - except Exception as e: - logger.debug( - "✗ %s background-task cancel error: %s", _p.value, e - ) - await self._safe_adapter_disconnect(_a, _p) - # Tear down adapters whose connect already succeeded — they - # were never registered, so stop() won't reach them. - for _t, (_p, _c, _a) in _task_map.items(): - if _t in _pending_tasks or _t.cancelled(): - continue - _res = _t.exception() is None and _t.result() or None - if _res and _res[3] == "ok": - try: - await _a.cancel_background_tasks() - except Exception as e: - logger.debug( - "✗ %s background-task cancel error: %s", - _p.value, e, - ) - await self._safe_adapter_disconnect(_a, _p) - await self._abort_startup_if_shutdown_requested() - return None - _raw = [ - _t.exception() or _t.result() for _t in _task_map - ] + if not _pending_connects: + return [] + # Abort-aware concurrent wait (parity with the serial loop's between-platforms check): a + # restart/shutdown requested mid-connect must cancel still-pending connects, clean up the + # ones already completed, and abort startup. + _task_map = { + asyncio.ensure_future(_connect_one_startup(p, c, a)): (p, c, a) + for (p, c, a) in _pending_connects + } + _pending_tasks = set(_task_map) + while _pending_tasks: + _done, _pending_tasks = await asyncio.wait(_pending_tasks, timeout=0.05) + if _pending_tasks and self._startup_should_abort(): + break else: - _raw = [] - return _raw + return [_t.exception() or _t.result() for _t in _task_map] + # Cancel and fully settle the in-flight connects FIRST, so a completed adapter's + # disconnect cannot unblock a sibling's connect() before the sibling is cancelled. + for _t in _pending_tasks: + _t.cancel() + await asyncio.gather(*_pending_tasks, return_exceptions=True) + # Then tear down adapters whose connect already succeeded — they were never registered, so + # stop() won't reach them. + _connected_ok = [ + _t for _t in _task_map + if _t not in _pending_tasks and not _t.cancelled() + and _t.exception() is None and _t.result()[3] == "ok" + ] + for _t in [*_pending_tasks, *_connected_ok]: + _p, _c, _a = _task_map[_t] + await self._startup_teardown_adapter(_a, _p) + await self._abort_startup_if_shutdown_requested() + return None + + def _startup_queue_transient_failure( + self, platform, adapter, platform_config, message: str, startup_retryable_errors: list + ) -> None: + """Mark a platform ``retrying`` with ``message`` and queue it in ``_failed_platforms``.""" + self._update_platform_runtime_status( + platform.value, platform_state="retrying", error_code=None, error_message=message, + ) + startup_retryable_errors.append(f"{platform.value}: {message}") + self._failed_platforms[platform] = self._startup_retry_entry(platform, adapter, platform_config) async def _start_aggregate_connect_results( - self, - _raw: list, - startup_retryable_errors: list, - startup_nonretryable_errors: list, + self, _raw: list, startup_retryable_errors: list, startup_nonretryable_errors: list ) -> int: - """Apply connect outcomes to shared state; returns the connected adapter count.""" + """Apply connect outcomes to shared state single-threaded (exactly as the original serial + loop did); returns the connected adapter count.""" connected_count = 0 - # Aggregate results single-threaded so shared state (self.adapters, self._failed_platforms, - # the error lists, connected_count) is mutated exactly as the original serial loop did -- - # only the connect() wall-clock overlap changed. for _item in _raw: if isinstance(_item, Exception): - # Unexpected escape from _connect_one_startup (shouldn't happen); - # log and skip rather than aborting the whole startup. + # Unexpected escape from _connect_one_startup (shouldn't happen); log and skip. logger.error("Unexpected startup connect error: %s", _item) continue platform, adapter, platform_config, outcome, exc = _item @@ -1407,228 +1102,164 @@ class GatewayStartupMixin: if outcome == "exception": logger.error("\u2717 %s error: %s", platform.value, exc) # Same defensive cleanup path for exceptions -- an adapter that raised mid-connect - # may still have a live aiohttp.ClientSession or child subprocess. + # may still have a live aiohttp.ClientSession or child subprocess. Unexpected + # exceptions are typically transient -- queue for retry. await self._safe_adapter_disconnect(adapter, platform) - self._update_platform_runtime_status( - platform.value, platform_state="retrying", error_code=None, error_message=str(exc), + self._startup_queue_transient_failure( + platform, adapter, platform_config, str(exc), startup_retryable_errors ) - startup_retryable_errors.append(f"{platform.value}: {exc}") - # Unexpected exceptions are typically transient -- queue for retry - self._failed_platforms[platform] = { - "config": platform_config, - "attempts": 1, - "next_retry": time.monotonic() + 30, - "queued_at": time.monotonic(), - "credential_claim": self._adapter_credential_claim(platform, adapter), - "listener_claim": self._adapter_listener_claim(platform, adapter), - } continue if outcome == "ok": self.adapters[platform] = adapter self._sync_voice_mode_state_to_adapter(adapter) - # Wire voice input callback at connect time so voice - # transcription is forwarded without requiring /voice join. + # Wire voice input at connect time so transcription works without /voice join. self._bind_voice_input_callback(adapter) connected_count += 1 self._update_platform_runtime_status( platform.value, platform_state="connected", error_code=None, error_message=None, ) logger.info("\u2713 %s connected", platform.value) - else: # outcome == "failed" - logger.warning("\u2717 %s failed to connect", platform.value) - # Defensive cleanup: a failed connect() may have allocated resources - # (aiohttp.ClientSession, poll tasks, bridge subprocesses) before giving up. - await self._safe_adapter_disconnect(adapter, platform) - if adapter.has_fatal_error: - # A live foreign holder of this bot token is a single-writer ownership conflict, - # not a blip — ``_acquire_platform_lock`` emits it retryable only so a MID-RUN - # reconnect can recover. At startup route it non-retryable: with nothing connected - # the gateway exits 78 instead of sitting alive and deaf in the retry queue. - _retryable = adapter.fatal_error_retryable and not ( - is_global_startup_conflict(adapter.fatal_error_code) - ) - self._update_platform_runtime_status( - platform.value, - platform_state="retrying" if _retryable else "fatal", - error_code=adapter.fatal_error_code, - error_message=adapter.fatal_error_message, - ) - target = ( - startup_retryable_errors - if _retryable - else startup_nonretryable_errors - ) - target.append(f"{platform.value}: {adapter.fatal_error_message}") - # Queue for reconnection if the error is retryable - if _retryable: - self._failed_platforms[platform] = { - "config": platform_config, - "attempts": 1, - "next_retry": time.monotonic() + 30, - "credential_claim": self._adapter_credential_claim(platform, adapter), - "listener_claim": self._adapter_listener_claim(platform, adapter), - } - else: - self._update_platform_runtime_status( - platform.value, platform_state="retrying", error_code=None, error_message="failed to connect", - ) - startup_retryable_errors.append(f"{platform.value}: failed to connect") - # No fatal error info means likely a transient issue -- queue for retry - self._failed_platforms[platform] = { - "config": platform_config, - "attempts": 1, - "next_retry": time.monotonic() + 30, - "queued_at": time.monotonic(), - "credential_claim": self._adapter_credential_claim(platform, adapter), - "listener_claim": self._adapter_listener_claim(platform, adapter), - } + continue + # outcome == "failed" + logger.warning("\u2717 %s failed to connect", platform.value) + # Defensive cleanup: a failed connect() may have allocated resources + # (aiohttp.ClientSession, poll tasks, bridge subprocesses) before giving up. + await self._safe_adapter_disconnect(adapter, platform) + if not adapter.has_fatal_error: + # No fatal error info means likely a transient issue -- queue for retry + self._startup_queue_transient_failure( + platform, adapter, platform_config, "failed to connect", startup_retryable_errors + ) + continue + # A live foreign holder of this bot token is a single-writer ownership conflict, not a + # blip — ``_acquire_platform_lock`` emits it retryable only so a MID-RUN reconnect can + # recover. At startup route it non-retryable: with nothing connected the gateway exits + # 78 instead of sitting alive and deaf in the retry queue. + _retryable = adapter.fatal_error_retryable and not ( + is_global_startup_conflict(adapter.fatal_error_code) + ) + self._update_platform_runtime_status( + platform.value, platform_state="retrying" if _retryable else "fatal", + error_code=adapter.fatal_error_code, error_message=adapter.fatal_error_message, + ) + target = startup_retryable_errors if _retryable else startup_nonretryable_errors + target.append(f"{platform.value}: {adapter.fatal_error_message}") + if _retryable: + self._failed_platforms[platform] = self._startup_retry_entry( + platform, adapter, platform_config, queued=False + ) return connected_count + def _startup_fail_fatal_config(self, reason: str) -> None: + """Record a fatal-config startup failure (exit 78) and request a clean exit.""" + from gateway.run import _write_runtime_status_quiet + _write_runtime_status_quiet(gateway_state="startup_failed", exit_reason=reason) + self._exit_code = GATEWAY_FATAL_CONFIG_EXIT_CODE + self._request_clean_exit(reason) + self._startup_restore_in_progress = False + async def _start_secondary_profiles( self, connected_count: int, _multiplex_skipped_platforms: list ) -> Tuple[bool, int]: """Bring up multiplexed secondary-profile adapters. Returns (aborted, connected_count).""" - from gateway.run import MultiplexConfigError, _write_runtime_status_quiet - # Multi-profile multiplexing: bring up adapters for every OTHER profile this gateway serves. - # Each profile's adapters connect under that profile's home + credential scope and stamp - # their inbound events with the profile so the agent turn resolves correctly. + from gateway.run import MultiplexConfigError + # Adapters for every OTHER profile this gateway serves connect under that profile's home + + # credential scope and stamp inbound events with the profile so the agent turn resolves. try: - _secondary_connected = await self._start_secondary_profile_adapters() - connected_count += _secondary_connected + connected_count += await self._start_secondary_profile_adapters() except MultiplexConfigError as e: - # Invalid multiplexer config — abort startup cleanly so the operator - # fixes config.yaml rather than running a half-wired gateway. - reason = str(e) - logger.error("Gateway multiplexer config error: %s", reason) - _write_runtime_status_quiet(gateway_state="startup_failed", exit_reason=reason) - self._exit_code = GATEWAY_FATAL_CONFIG_EXIT_CODE - self._request_clean_exit(reason) - self._startup_restore_in_progress = False + # Invalid multiplexer config — abort startup cleanly so the operator fixes config.yaml + # rather than running a half-wired gateway. + logger.error("Gateway multiplexer config error: %s", str(e)) + self._startup_fail_fatal_config(str(e)) return True, connected_count except Exception as e: logger.error("Secondary-profile adapter startup failed: %s", e, exc_info=True) finally: - # Startup authority is one phase, not a persistent runner mode. - # From this point onward every adapter retry is non-evicting. + # Startup authority is one phase, not a persistent runner mode: from here on every + # adapter retry is non-evicting. self._platform_lock_takeover_on_start = False # A platform skipped on the primary for a missing credential should have been picked up by # a secondary profile owning the token. If none did, it is enabled in config.yaml yet # silently unserved — surface it loudly instead of leaving a quiet dead channel. for _skipped in _multiplex_skipped_platforms: - _served_by_secondary = any( - _skipped in _profile_map - for _profile_map in self._profile_adapters.values() - ) - if not _served_by_secondary: + if not any(_skipped in _profile_map for _profile_map in self._profile_adapters.values()): logger.warning( "%s is enabled but no profile (default or secondary) " "provided a bot credential for it — the platform is not " "being served. Add its token to the profile that should " - "own it, or disable the platform.", - _skipped.value, + "own it, or disable the platform.", _skipped.value, ) return False, connected_count def _start_handle_no_connections( - self, - connected_count: int, - enabled_platform_count: int, - startup_retryable_errors: list, + self, connected_count: int, enabled_platform_count: int, startup_retryable_errors: list, startup_nonretryable_errors: list, ) -> bool: """Log/degrade when nothing connected; return True when startup must exit.""" - from gateway.run import _write_runtime_status_quiet - if connected_count == 0: - if startup_nonretryable_errors and not startup_retryable_errors: - reason = "; ".join(startup_nonretryable_errors) - logger.error("Gateway hit a non-retryable startup conflict: %s", reason) - _write_runtime_status_quiet(gateway_state="startup_failed", exit_reason=reason) - self._exit_code = GATEWAY_FATAL_CONFIG_EXIT_CODE - self._request_clean_exit(reason) - self._startup_restore_in_progress = False - return True - if startup_nonretryable_errors: - # Mixed failure mode: some platforms fatally misconfigured (e.g. WhatsApp never - # paired), others merely transient (e.g. Telegram TimedOut). Exiting 78 here would - # let exit-78 supervisors take the gateway PERMANENTLY down over a network blip and - # deny the retryable ones their retry. Log the fatal side loudly, then fall through to - # the degraded/retry path: the watcher recovers the retryable; the rest stay parked. - logger.error( - "%d platform(s) fatally misconfigured and parked: %s. " - "Staying alive so retryable platforms can recover.", - len(startup_nonretryable_errors), - "; ".join(startup_nonretryable_errors), - ) - if enabled_platform_count > 0: - if startup_retryable_errors: - # All enabled platforms hit retryable failures (network blip, bridge not paired, - # npm install timeout...). Keep the gateway alive so cron jobs still run and the - # reconnect watcher can recover the platforms once the cause is fixed; exiting - # here would turn one misconfigured platform into an infinite systemd restart loop. - reason = "; ".join(startup_retryable_errors) - logger.warning( - "Gateway started with no connected platforms — " - "%d platform(s) queued for retry: %s", - len(self._failed_platforms), reason, - ) - try: - from gateway.status import write_runtime_status - write_runtime_status( - gateway_state="degraded", - exit_reason=None, - ) - except Exception: - pass - # Fall through to the normal "running" state — reconnect watcher takes it from here. - # All enabled platforms had no adapter (missing library or credentials). Fleet nodes - # share one config.yaml but hold credentials for only a subset of platforms, so - # degrade gracefully and let cron jobs run. - logger.warning( - "No adapter could be created for any of the %d configured platform(s). " - "Check that required dependencies are installed and credentials are set. " - "Gateway will continue for cron job execution.", - enabled_platform_count, - ) - else: - logger.warning("No messaging platforms enabled.") - logger.info("Gateway will continue running for cron job execution.") + if connected_count != 0: + return False + if startup_nonretryable_errors and not startup_retryable_errors: + reason = "; ".join(startup_nonretryable_errors) + logger.error("Gateway hit a non-retryable startup conflict: %s", reason) + self._startup_fail_fatal_config(reason) + return True + if startup_nonretryable_errors: + # Mixed failure mode (some fatal, some transient). Exiting 78 here would let exit-78 + # supervisors take the gateway PERMANENTLY down over a network blip and deny the + # retryable ones their retry. Log the fatal side loudly, then fall through to the + # degraded/retry path: the watcher recovers the retryable; the rest stay parked. + logger.error( + "%d platform(s) fatally misconfigured and parked: %s. " + "Staying alive so retryable platforms can recover.", + len(startup_nonretryable_errors), "; ".join(startup_nonretryable_errors), + ) + if enabled_platform_count <= 0: + logger.warning("No messaging platforms enabled.") + logger.info("Gateway will continue running for cron job execution.") + return False + if startup_retryable_errors: + # All enabled platforms hit retryable failures. Keep the gateway alive so cron jobs + # still run and the reconnect watcher can recover them once the cause is fixed; + # exiting would turn one misconfigured platform into an infinite systemd restart loop. + logger.warning( + "Gateway started with no connected platforms — %d platform(s) queued for retry: %s", + len(self._failed_platforms), "; ".join(startup_retryable_errors), + ) + with suppress(Exception): + from gateway.status import write_runtime_status + write_runtime_status(gateway_state="degraded", exit_reason=None) + # (Or: no adapter for any enabled platform — missing library or credentials.) Fleet nodes + # share one config.yaml but hold credentials for only a subset, so degrade gracefully. + logger.warning( + "No adapter could be created for any of the %d configured platform(s). " + "Check that required dependencies are installed and credentials are set. " + "Gateway will continue for cron job execution.", enabled_platform_count, + ) return False - async def _start_finish_wiring(self, connected_count: int) -> None: - """Post-connect wiring: room worker, heartbeat, hooks, notifications, restore, watchers.""" - from gateway.run import ( - _hermes_home, - _planned_restart_notification_pending, - _restart_notification_pending, - ) + async def _start_post_connect_services(self, connected_count: int) -> None: + """Room worker, heartbeat, gateway:startup hook, channel directory, /update notice.""" + from gateway.run import _hermes_home try: await self._ensure_hosted_room_worker() except Exception: logger.error( "Group Chat worker failed to start; mutating Group Chat commands " - "will fail closed until supervision recovers it", - exc_info=True, + "will fail closed until supervision recovers it", exc_info=True, ) - self._spawn_supervised( - self._hosted_room_worker_watcher, - "hosted_room_worker", - ) - + self._spawn_supervised(self._hosted_room_worker_watcher, "hosted_room_worker") self._start_loop_heartbeat_task() - # Emit gateway:startup hook hook_count = len(self.hooks.loaded_hooks) if hook_count: logger.info("%s hook(s) loaded", hook_count) - await self.hooks.emit("gateway:startup", { - "platforms": [p.value for p in self.adapters], - }) - + await self.hooks.emit("gateway:startup", {"platforms": [p.value for p in self.adapters]}) if connected_count > 0: logger.info("Gateway running with %s platform(s)", connected_count) - # Build initial channel directory for send_message name resolution + # Initial channel directory for send_message name resolution try: from gateway.channel_directory import build_channel_directory directory = await build_channel_directory(self.adapters) @@ -1637,37 +1268,36 @@ class GatewayStartupMixin: except Exception as e: logger.warning("Channel directory build failed: %s", e) - # Check if we're restarting after a /update command. If the update is - # still running, keep watching so we notify once it actually finishes. + # Restarting after a /update? If the update is still running, keep watching so we notify + # once it actually finishes. notified = await self._send_update_notification() if not notified and any( - path.exists() - for path in ( - _hermes_home / ".update_pending.json", - _hermes_home / ".update_pending.claimed.json", - ) + (_hermes_home / name).exists() + for name in (".update_pending.json", ".update_pending.claimed.json") ): self._schedule_update_notification_watch() + async def _start_finish_wiring(self, connected_count: int) -> None: + """Post-connect wiring: services, boot notifications, startup restore, recovered watchers.""" + from gateway.run import _planned_restart_notification_pending, _restart_notification_pending + await self._start_post_connect_services(connected_count) + # Give freshly connected adapters a brief moment to settle before sending restart/startup # lifecycle messages; in practice this helps Discord thread deliveries after reconnect. if connected_count > 0: await asyncio.sleep(1.0) - # Notify the chat that initiated /restart that the gateway is back. - chat_restart_notification_pending = _restart_notification_pending() - planned_restart_notification_pending = _planned_restart_notification_pending() # Capture, before _send_restart_notification() unlinks the marker, whether this process # booted from a chat-originated /restart. One-shot signal for the /restart redelivery # guard (_is_stale_restart_redelivery): a missing dedup marker only suppresses a /restart # when we KNOW we just came out of a restart cycle. - if chat_restart_notification_pending: + if _restart_notification_pending(): self._booted_from_restart = True # Restart notification, home-channel startup notice, and obligation redelivery all call # adapter.send(). Those sends must not pin the inbound restore gate — a Telegram flood- # control sleep on this path froze every platform for the full penalty. await self._await_startup_boot_sends( - planned_restart_notification_pending=planned_restart_notification_pending, + planned_restart_notification_pending=_planned_restart_notification_pending(), ) # Auto-continue fresh sessions interrupted by the previous restart/shutdown. resume_pending @@ -1676,25 +1306,22 @@ class GatewayStartupMixin: self._schedule_resume_pending_sessions() await self._finish_startup_restore() - # Surface state.db init failures to the user's messaging platforms - # so they know persistence is broken before losing data (#88235). + # Surface state.db init failures to the user's messaging platforms so they know persistence + # is broken before losing data. await self._send_session_db_warning_notifications() - # Drain any recovered process watchers (from crash recovery checkpoint) + # Drain recovered process watchers (crash-recovery checkpoint). Detach the batch atomically: + # reassigning a fresh list takes ownership of exactly the watchers present now, so one + # appended concurrently during the yield below isn't dropped by a clear() on the shared + # list. Yield every 100 to avoid O(n^2) loop blocking when recovering thousands. try: from tools.process_registry import process_registry - # Detach the current batch atomically: reassigning to a fresh list takes ownership of - # exactly the watchers present now, so any watcher appended concurrently during the - # yield below isn't silently dropped by a clear() on the shared list. watchers = process_registry.pending_watchers process_registry.pending_watchers = [] - # Process in batches of 100 with event-loop yield points to avoid - # O(n^2) event-loop blocking when recovering thousands of watchers. for i, watcher in enumerate(watchers): self._spawn_supervised( lambda w=watcher: self._run_process_watcher(w), - f"process_watcher:{watcher.get('session_id')}", - restart=False, + f"process_watcher:{watcher.get('session_id')}", restart=False, ) logger.info("Resumed watcher for recovered process %s", watcher.get("session_id")) if i % 100 == 99: @@ -1702,60 +1329,39 @@ class GatewayStartupMixin: except Exception as e: logger.error("Recovered watcher setup error: %s", e) + # Long-lived supervised watchers spawned at the end of start(), in order; the supervised name is + # the method name without its leading underscore. Each watcher's docstring explains its job + # (session expiry, /model catalog TTL refresh, stall warnings, kanban notifier/dispatcher, + # CLI handoff, async-delegation completions, /loop wakeups). + _PRE_RECONNECT_WATCHERS = ( + "_session_expiry_watcher", "_model_catalog_refresh_watcher", "_session_stall_watcher", + "_kanban_notifier_watcher", "_kanban_dispatcher_watcher", + ) + _POST_RECONNECT_WATCHERS = ("_handoff_watcher", "_async_delegation_watcher", "_loop_wakeup_watcher") + def _start_spawn_background_watchers(self) -> None: """Spawn the long-lived supervised background watchers.""" - # Start background session expiry watcher to finalize expired sessions - self._spawn_supervised(self._session_expiry_watcher, "session_expiry_watcher") + for method in self._PRE_RECONNECT_WATCHERS: + self._spawn_supervised(getattr(self, method), method[1:]) - # Keep the /model picker's remote catalogs (curated manifest, OpenRouter live list, Nous - # Portal recommendations) warm on disk so a delisted or newly-published model reaches the - # picker within one TTL window (model_catalog.ttl_minutes, default 20) without a cold open. - self._spawn_supervised(self._model_catalog_refresh_watcher, "model_catalog_refresh_watcher") - - # Stall watchdog: pending inbound + stale agent activity → warn user - # to /new (does not kill the turn; see agent.session_stall_timeout). - self._spawn_supervised(self._session_stall_watcher, "session_stall_watcher") - - # Start the kanban notifier — each gateway delivers events for subscriptions owned by the - # profiles whose adapters it hosts, even when another gateway owns the single dispatcher. - self._spawn_supervised(self._kanban_notifier_watcher, "kanban_notifier_watcher") - - # Start background kanban dispatcher — spawns workers for ready tasks. Gated by - # `kanban.dispatch_in_gateway` (default True). When false, users run `hermes kanban daemon` - # externally or simply don't use kanban; this loop becomes a no-op. - self._spawn_supervised(self._kanban_dispatcher_watcher, "kanban_dispatcher_watcher") - - # Start background reconnection watcher for platforms that failed at startup if self._failed_platforms: logger.info( "Starting reconnection watcher for %d failed platform(s): %s", - len(self._failed_platforms), - ", ".join(p.value for p in self._failed_platforms), + len(self._failed_platforms), ", ".join(p.value for p in self._failed_platforms), ) - # Track the reconnect watcher task so _ensure_reconnect_watcher_running can detect death - # and respawn it. Spawned via _spawn_supervised so an exception escaping the watcher's OUTER - # loop is caught, logged, and restarted with backoff instead of silently killing it (else a - # platform already queued in _failed_platforms stays stranded: the ensure hook only runs on - # a NEW fatal-error arrival). ``on_spawn`` keeps ``_reconnect_watcher_task`` on the CURRENT - # live task across backoff respawns so a superseded handle never looks like a dead watcher. + # Spawned via _spawn_supervised so an exception escaping the watcher's OUTER loop is caught, + # logged, and restarted with backoff instead of silently killing it (else a platform already + # queued in _failed_platforms stays stranded: the ensure hook only runs on a NEW fatal-error + # arrival). ``on_spawn`` keeps ``_reconnect_watcher_task`` on the CURRENT live task across + # backoff respawns so a superseded handle never looks like a dead watcher. self._spawn_reconnect_watcher() - # Start background handoff watcher — picks up CLI sessions marked handoff_state='pending' in - # state.db and re-binds them to the destination platform's home channel, then forges a - # synthetic user turn so the agent kicks off the new chat. - self._spawn_supervised(self._handoff_watcher, "handoff_watcher") + for method in self._POST_RECONNECT_WATCHERS: + self._spawn_supervised(getattr(self, method), method[1:]) - # Async-delegation watcher: drains delegate_task(background=true) completions and injects - # each result into its originating session as a new turn (covers the idle, no-turn case). - self._spawn_supervised(self._async_delegation_watcher, "async_delegation_watcher") - - # /loop wakeup watcher: scans persisted loops (SessionDB loop:* rows) and injects due - # wakeup prompts into their originating chats while the session is idle. - self._spawn_supervised(self._loop_wakeup_watcher, "loop_wakeup_watcher") - - # Start the scale-to-zero idle watcher ONLY when opted in (HERMES_SCALE_TO_ZERO stamp), - # messaging is relay-only/absent, and a wakeUrl is registered. When armed it drives the relay - # dormant on sustained idle, then suspends via flaps — Fly autostop is inbound-only, job-blind. + # Scale-to-zero idle watcher ONLY when opted in (HERMES_SCALE_TO_ZERO stamp), messaging is + # relay-only/absent, and a wakeUrl is registered. When armed it drives the relay dormant on + # sustained idle, then suspends via flaps — Fly autostop is inbound-only, job-blind. try: if self._scale_to_zero_should_arm(): logger.info( @@ -1776,10 +1382,7 @@ class GatewayStartupMixin: self._spawn_supervised(self._drain_control_watcher, "drain_control_watcher") async def start(self) -> bool: - """Start the gateway and all configured platform adapters. - - Returns True if at least one adapter connected successfully. - """ + """Start the gateway and all configured platform adapters.""" logger.info("Starting Hermes Gateway...") self._start_install_faulthandler() self._start_log_startup_environment() @@ -1803,10 +1406,7 @@ class GatewayStartupMixin: startup_nonretryable_errors: list[str] = [] startup_retryable_errors: list[str] = [] ( - _aborted, - enabled_platform_count, - _multiplex_skipped_platforms, - _pending_connects, + _aborted, enabled_platform_count, _multiplex_skipped_platforms, _pending_connects ) = await self._start_prefilter_platforms() if _aborted: return True @@ -1828,14 +1428,11 @@ class GatewayStartupMixin: if _aborted: return True if self._start_handle_no_connections( - connected_count, - enabled_platform_count, - startup_retryable_errors, + connected_count, enabled_platform_count, startup_retryable_errors, startup_nonretryable_errors, ): return True - # Update delivery router with adapters if await self._abort_startup_if_shutdown_requested(): return True self.delivery_router.adapters = self.adapters @@ -1846,11 +1443,137 @@ class GatewayStartupMixin: self._update_runtime_status("running") await self._start_finish_wiring(connected_count) self._start_spawn_background_watchers() - logger.info("Press Ctrl+C to stop") - return True + @dataclasses.dataclass + class _HandoffDestination: + """Resolved destination for one handoff row.""" + platform: Platform + platform_name: str + transport: Any + home: Any + home_chat_id: str + effective_thread_id: Optional[str] + source: SessionSource + handoff_config: Any + + def _handoff_resolve_scope(self, profile_name: Optional[str]): + """Return (config, adapters) for the profile that queued the handoff. For a secondary + profile the watcher already entered _profile_runtime_scope, so a fresh load resolves THAT + profile's config; fail closed — self.config would deliver to the WRONG chat.""" + from gateway.run import load_gateway_config + if not profile_name or profile_name == "default": + return self.config, self.adapters + secondary = (self._profile_adapters or {}).get(profile_name) + if not secondary: + raise RuntimeError(f"profile '{profile_name}' has no live adapters in this gateway") + try: + return load_gateway_config(), secondary + except Exception as exc: + logger.error( + "Handoff: could not load config for profile %s; failing the handoff instead of " + "delivering via the primary's config", profile_name, exc_info=True, + ) + raise RuntimeError(f"could not load config for profile '{profile_name}': {exc}") from exc + + async def _handoff_resolve_destination( + self, row: Dict[str, Any], profile_name: Optional[str] + ) -> "GatewayStartupMixin._HandoffDestination": + """Resolve platform, transport, home channel, thread and destination source for a row.""" + from gateway.run import resolve_delivery_transport + cli_session_id = row["id"] + platform_name = (row.get("handoff_platform") or "").strip().lower() + if not platform_name: + raise RuntimeError("handoff_platform is empty") + try: + platform = Platform(platform_name) + except (ValueError, KeyError): + raise RuntimeError(f"unknown platform '{platform_name}'") + + handoff_config, handoff_adapters = self._handoff_resolve_scope(profile_name) + + # Adapter must be live. A relay-fronted gateway registers ONE adapter under Platform.RELAY + # fronting N logical platforms, so a literal adapters.get(discord) misses a deliverable + # platform; resolve_delivery_transport is the alias-aware resolver (native adapter wins). + transport = resolve_delivery_transport(platform, handoff_config, handoff_adapters) + if not transport: + raise RuntimeError(f"platform '{platform_name}' is not active in this gateway") + home = handoff_config.get_home_channel(platform) + if not home or not home.chat_id: + raise RuntimeError( + f"no home channel configured for {platform_name}; run /sethome on the desired chat first" + ) + home_chat_id = str(home.chat_id) + + # Fresh thread on the destination so the handoff has its own scrollback. Adapter returns + # None if threading is unsupported (Matrix/WhatsApp/Signal/SMS) or creation failed. + cli_title = row.get("title") or cli_session_id[:8] + try: + new_thread_id = await transport.adapter.create_handoff_thread( + home_chat_id, f"Hermes — {cli_title}", + ) + except Exception as exc: + logger.debug( + "Handoff: create_handoff_thread raised on %s: %s", platform_name, exc, exc_info=True + ) + new_thread_id = None + effective_thread_id = new_thread_id or (str(home.thread_id) if home.thread_id else None) + + # Telegram private-chat DM topics are shaped differently from group/forum threads by the + # inbound adapter: a handoff-created topic in a positive chat_id must use the DM-topic source + # shape (real user id == chat_id, so topic-mode checks and binding persistence match later + # inbound turns), or the synthetic turn binds a `thread` key while replies arrive on `dm`. + is_telegram_private_chat = ( + platform == Platform.TELEGRAM and looks_like_telegram_private_chat_id(home_chat_id) + ) + is_thread = bool(new_thread_id) and not is_telegram_private_chat + # Discord (unlike Slack/Telegram) builds in-thread messages with ``chat_id == thread id``, + # so key on the thread's OWN id; keying on the parent would make the next reply spawn anew. + dest_source = SessionSource( + platform=platform, + chat_id=str(effective_thread_id) if ( + is_thread and platform == Platform.DISCORD and effective_thread_id + ) else home_chat_id, + chat_name=home.name, + chat_type="thread" if is_thread else "dm", + user_id=home_chat_id if is_telegram_private_chat else "system:handoff", + user_name="Handoff", thread_id=effective_thread_id, profile=profile_name, + ) + return self._HandoffDestination( + platform=platform, platform_name=platform_name, transport=transport, home=home, + home_chat_id=home_chat_id, effective_thread_id=effective_thread_id, source=dest_source, + handoff_config=handoff_config, + ) + + def _handoff_session_key(self, dest, profile_name: Optional[str]) -> str: + """Build the destination session_key with the adapters' own rules. + + Thread keys omit user_id (thread_sessions_per_user default) so the next message shares it. + The key is namespaced to the queuing profile: a multiplexed gateway would otherwise build + ``agent:main:...`` while the profile's adapter routes inbound on ``agent::...``. + The store resolver is only the root fallback (None when multiplexing is off; old key + unchanged). The isinstance check is load-bearing: a Mock store returns a truthy MagicMock. + """ + platform_cfg = dest.handoff_config.platforms.get(dest.platform) + extra = platform_cfg.extra if platform_cfg else {} + handoff_profile = profile_name if (profile_name and profile_name != "default") else None + if handoff_profile is None: + try: + store = getattr(self.async_session_store, "_store", self.async_session_store) + resolver = getattr(store, "_resolve_profile_for_key", None) + if callable(resolver): + resolved = resolver(dest.source) + if isinstance(resolved, str) and resolved.strip(): + handoff_profile = resolved + except Exception: + logger.debug("Handoff: could not resolve profile namespace", exc_info=True) + return build_session_key( + dest.source, group_sessions_per_user=extra.get("group_sessions_per_user", True), + thread_sessions_per_user=extra.get("thread_sessions_per_user", False), + profile=handoff_profile, + ) + async def _process_handoff( self, row: Dict[str, Any], profile_name: Optional[str] = None, ) -> None: @@ -1861,211 +1584,59 @@ class GatewayStartupMixin: live in ``_profile_adapters``), and the session key must be namespaced ``agent::...`` or it binds a key nobody reads. Passing the name beats re-deriving it from the contextvar. """ - from gateway.run import load_gateway_config, resolve_delivery_transport - from gateway.config import Platform - from gateway.session import SessionSource, build_session_key - from gateway.platforms.base import MessageEvent - cli_session_id = row["id"] - platform_name = (row.get("handoff_platform") or "").strip().lower() - if not platform_name: - raise RuntimeError("handoff_platform is empty") - - # Resolve platform enum - try: - platform = Platform(platform_name) - except (ValueError, KeyError): - raise RuntimeError(f"unknown platform '{platform_name}'") - - # Resolve the config + adapter map for the profile that queued this handoff; single-profile - # gateways (or a default-profile handoff) fall back to self.config/self.adapters. - handoff_config = self.config - handoff_adapters = self.adapters - if profile_name and profile_name != "default": - secondary = (self._profile_adapters or {}).get(profile_name) - if not secondary: - raise RuntimeError( - f"profile '{profile_name}' has no live adapters in this gateway" - ) - handoff_adapters = secondary - # The watcher already entered _profile_runtime_scope, so a fresh load resolves THIS - # profile's config. Fail closed — self.config would deliver to the WRONG chat. - try: - handoff_config = load_gateway_config() - except Exception as exc: - logger.error( - "Handoff: could not load config for profile %s; " - "failing the handoff instead of delivering via the " - "primary's config", - profile_name, exc_info=True, - ) - raise RuntimeError( - f"could not load config for profile '{profile_name}': {exc}" - ) from exc - - # Adapter must be live. A relay-fronted gateway registers ONE adapter under Platform.RELAY - # fronting N logical platforms, so a literal adapters.get(discord) misses a deliverable - # platform; resolve_delivery_transport is the alias-aware resolver (native adapter wins). - transport = resolve_delivery_transport(platform, handoff_config, handoff_adapters) - if not transport: - raise RuntimeError( - f"platform '{platform_name}' is not active in this gateway" - ) - adapter = transport.adapter - - # Home channel must be configured - home = handoff_config.get_home_channel(platform) - if not home or not home.chat_id: - raise RuntimeError( - f"no home channel configured for {platform_name}; " - f"run /sethome on the desired chat first" - ) - - cli_title = row.get("title") or cli_session_id[:8] - - # Create a fresh thread on the destination so the handoff has its own scrollback. Adapter - # returns None if threading is unsupported (Matrix/WhatsApp/Signal/SMS) or creation failed. - thread_name = f"Hermes — {cli_title}" - try: - new_thread_id = await adapter.create_handoff_thread( - str(home.chat_id), thread_name, - ) - except Exception as exc: - logger.debug( - "Handoff: create_handoff_thread raised on %s: %s", - platform_name, exc, exc_info=True, - ) - new_thread_id = None - - effective_thread_id = new_thread_id or ( - str(home.thread_id) if home.thread_id else None - ) - - # Telegram private-chat DM topics are shaped differently from group/forum threads by the - # inbound adapter: a handoff-created topic in a positive chat_id must use the DM-topic source - # shape, or the synthetic turn binds a `thread` key while real replies arrive on a `dm` key. - home_chat_id = str(home.chat_id) - is_telegram_private_chat = ( - platform == Platform.TELEGRAM - and looks_like_telegram_private_chat_id(home_chat_id) - ) - - if new_thread_id and not is_telegram_private_chat: - dest_chat_type = "thread" - dest_user_id = "system:handoff" - else: - # No thread — assume DM-style. For Telegram private-chat topics use the real user id - # (== chat_id) so topic-mode checks and binding persistence match later inbound turns. - dest_chat_type = "dm" - dest_user_id = home_chat_id if is_telegram_private_chat else "system:handoff" - - # Discord (unlike Slack/Telegram) builds in-thread messages with ``chat_id == thread id``, - # so key on the thread's OWN id; keying on the parent would make the next reply spawn anew. - if platform == Platform.DISCORD and dest_chat_type == "thread" and effective_thread_id: - dest_chat_id = str(effective_thread_id) - else: - dest_chat_id = home_chat_id - dest_source = SessionSource( - platform=platform, - chat_id=dest_chat_id, - chat_name=home.name, - chat_type=dest_chat_type, - user_id=dest_user_id, - user_name="Handoff", - thread_id=effective_thread_id, - profile=profile_name, - ) - - # Build the session_key with the adapters' own rules so switch_session hits the right entry. - # Thread keys omit user_id (thread_sessions_per_user default) so the next message shares it. - platform_cfg = handoff_config.platforms.get(platform) - extra = platform_cfg.extra if platform_cfg else {} - # Namespace the key to the queuing profile: a multiplexed gateway would otherwise build - # ``agent:main:...`` while the profile's adapter routes inbound on ``agent::...``. - # The resolver is only the root fallback (None when multiplexing is off; old key unchanged). - # The isinstance check is load-bearing: a Mock store returns a truthy MagicMock. - handoff_profile = profile_name if (profile_name and profile_name != "default") else None - if handoff_profile is None: - try: - store = getattr(self.async_session_store, "_store", self.async_session_store) - resolver = getattr(store, "_resolve_profile_for_key", None) - if callable(resolver): - resolved = resolver(dest_source) - if isinstance(resolved, str) and resolved.strip(): - handoff_profile = resolved - except Exception: - logger.debug("Handoff: could not resolve profile namespace", exc_info=True) - session_key = build_session_key( - dest_source, - group_sessions_per_user=extra.get("group_sessions_per_user", True), - thread_sessions_per_user=extra.get("thread_sessions_per_user", False), - profile=handoff_profile, - ) + dest = await self._handoff_resolve_destination(row, profile_name) + session_key = self._handoff_session_key(dest, profile_name) # Ensure a session_store entry exists for this key (get_or_create_session creates one for a # never-used home channel); switch_session then re-points it. - await self.async_session_store.get_or_create_session(dest_source) - + await self.async_session_store.get_or_create_session(dest.source) # Re-bind the destination key to the CLI session_id: switch_session ends the prior session # in SQLite and reopens the CLI session under the new key; its transcript is now active. switched = await self.async_session_store.switch_session(session_key, cli_session_id) if switched is None: - raise RuntimeError( - f"could not switch session key {session_key} → {cli_session_id}" - ) - - # Evict any cached AIAgent for this session_key so the next dispatch - # rebuilds it against the CLI session_id (mirrors /resume / /branch). + raise RuntimeError(f"could not switch session key {session_key} → {cli_session_id}") + # Evict any cached AIAgent for this key so the next dispatch rebuilds it against the CLI + # session_id (mirrors /resume / /branch), and clear stale running-agent state so the + # synthetic turn isn't queued behind it. self._evict_cached_agent(session_key) - - # Cancel any in-flight running-agent state for the destination key - # so the synthetic turn isn't queued behind a stale running flag. self._release_running_agent_state(session_key) - synthetic_text = ( - f"[Session was just handed off from CLI (\"{cli_title}\") to this " - f"channel. The full prior conversation history is loaded above. " - f"Briefly confirm you're working here and summarize what we were " - f"working on, so the user can continue from this device.]" - ) - + cli_title = row.get("title") or cli_session_id[:8] synthetic_event = MessageEvent( - text=synthetic_text, - source=dest_source, + text=( + f"[Session was just handed off from CLI (\"{cli_title}\") to this " + f"channel. The full prior conversation history is loaded above. " + f"Briefly confirm you're working here and summarize what we were " + f"working on, so the user can continue from this device.]" + ), + source=dest.source, internal=True, ) - logger.info( "Handoff: dispatching synthetic turn for CLI session %s → %s " "(home=%s, thread=%s, session_key=%s)", - cli_session_id, platform_name, home.chat_id, effective_thread_id, + cli_session_id, dest.platform_name, dest.home.chat_id, dest.effective_thread_id, session_key, ) - # Dispatch through the runner directly: adapter.handle_message would spawn a background task # and lose error visibility; inline _handle_message keeps success/failure observable. response_text = await self._handle_message(synthetic_event) if not response_text: - # Streaming may have already delivered the response inline. - # Either way, agent ran without raising — count as success. + # Streaming may have already delivered the response inline; the agent ran without + # raising either way — count as success. return # Send the reply to the new thread if we created one, else the configured home channel # (which may carry a thread_id). Use the resolved transport (not adapter.send) so a # relay-fronted logical platform is stamped on the outbound frame (send_for_platform). - send_metadata: Dict[str, Any] = {} - if effective_thread_id: - send_metadata["thread_id"] = effective_thread_id + send_metadata = {"thread_id": dest.effective_thread_id} if dest.effective_thread_id else None try: - result = await transport.send( - platform, - str(home.chat_id), - response_text, - send_metadata or None, + result = await dest.transport.send( + dest.platform, str(dest.home.chat_id), response_text, send_metadata, ) except Exception as exc: raise RuntimeError(f"adapter.send failed: {exc}") from exc - if not getattr(result, "success", True): err = getattr(result, "error", "send returned success=False") raise RuntimeError(f"adapter.send failed: {err}") diff --git a/gateway/run_topics.py b/gateway/run_topics.py index 9ee08c773a..7268ecd739 100644 --- a/gateway/run_topics.py +++ b/gateway/run_topics.py @@ -7,18 +7,19 @@ so ``patch("gateway.run.X")`` keeps intercepting them at call time. from __future__ import annotations -import logging -from typing import TYPE_CHECKING import asyncio import dataclasses +import logging import re +import time +from pathlib import Path +from typing import TYPE_CHECKING, Optional, Tuple + from agent.compaction_display import project_compaction_message_for_display from agent.i18n import t from gateway.config import Platform from gateway.platforms.base import MessageEvent, _prefix_within_utf16_limit, utf16_len from gateway.session import SessionSource -from pathlib import Path -from typing import Optional, Tuple if TYPE_CHECKING: # string annotations only; never imported at runtime (cycle) from gateway.run import GatewayRunner, TurnRunner # noqa: F401 @@ -26,10 +27,17 @@ if TYPE_CHECKING: # string annotations only; never imported at runtime (cycle) # Log-record parity with the origin module. logger = logging.getLogger("gateway.run") +_TOPIC_RESTORE_STEPS = ( + "1. Create or open a topic. To create a new one, open All Messages and send any message there.", + "2. Send /topic inside that topic.", +) + class GatewayTopicThreadsMixin: """Telegram forum-topic and Discord auto-thread binding/rename methods for GatewayRunner.""" + # ── Telegram topic mode: predicates and keys ──────────────────────────────────────────── + @staticmethod def _telegram_topic_profile_name(source: SessionSource) -> str: """Profile namespace for Telegram topic-mode rows. @@ -40,72 +48,85 @@ class GatewayTopicThreadsMixin: name = str(getattr(source, "profile", None) or "").strip() return name if name else "default" + def _sync_session_db(self): + """The sync SessionDB handle, or None. Only for callers that provably run off-loop + (asyncio.to_thread / the run_sync executor).""" + session_db = getattr(self, "_session_db", None) + return None if session_db is None else getattr(session_db, "_db", session_db) + + @staticmethod + def _is_telegram_dm(source: SessionSource) -> bool: + return source.platform == Platform.TELEGRAM and source.chat_type == "dm" + def _telegram_topic_mode_enabled(self, source: SessionSource) -> bool: """Return whether Telegram DM topic mode is active for this chat.""" - if source.platform != Platform.TELEGRAM or source.chat_type != "dm": + if not self._is_telegram_dm(source): return False - session_db = getattr(self, "_session_db", None) + session_db = self._sync_session_db() if session_db is None: return False - # Runs off-loop (always via asyncio.to_thread); use the sync handle. - session_db = getattr(session_db, "_db", session_db) try: raw = session_db.is_telegram_topic_mode_enabled( - chat_id=str(source.chat_id), - user_id=str(source.user_id), + chat_id=str(source.chat_id), user_id=str(source.user_id), profile_name=self._telegram_topic_profile_name(source), ) except Exception: logger.debug("Failed to read Telegram topic mode state", exc_info=True) return False - # Only a real True from the SessionDB enables topic mode; anything else (including MagicMock - # from test fixtures that didn't opt in) means off for this chat. + # Only a real True enables topic mode; anything else (including MagicMock from test + # fixtures that didn't opt in) means off for this chat. return raw is True def _is_telegram_topic_root_lobby(self, source: SessionSource) -> bool: """True for the main Telegram DM (or General topic) when topic mode has made it a lobby.""" - if source.platform != Platform.TELEGRAM or source.chat_type != "dm": + if not self._is_telegram_dm(source) or not self._telegram_topic_mode_enabled(source): return False - if not self._telegram_topic_mode_enabled(source): - return False - tid = str(source.thread_id or "") - return tid in self._TELEGRAM_GENERAL_TOPIC_IDS + return str(source.thread_id or "") in self._TELEGRAM_GENERAL_TOPIC_IDS def _is_telegram_topic_lane(self, source: SessionSource) -> bool: """True for a user-created Telegram private-chat topic lane.""" - if source.platform != Platform.TELEGRAM or source.chat_type != "dm": - return False - if not self._telegram_topic_mode_enabled(source): + if not self._is_telegram_dm(source) or not self._telegram_topic_mode_enabled(source): return False tid = str(source.thread_id or "") return bool(tid) and tid not in self._TELEGRAM_GENERAL_TOPIC_IDS def _telegram_topic_cooldown_key(self, source: SessionSource) -> Optional[str]: - """Cooldown key for topic-mode cooldowns: (profile, chat_id). - - Profiles sharing a Telegram private chat_id under multiplex must not - suppress each other's lobby reminders / capability hints (#76423). - """ + """Cooldown key (profile, chat_id): profiles sharing a Telegram private chat_id under + multiplex must not suppress each other's lobby reminders / capability hints.""" chat_id = str(source.chat_id or "") if not chat_id: return None return f"{self._telegram_topic_profile_name(source)}:{chat_id}" - def _should_send_telegram_lobby_reminder(self, source: SessionSource) -> bool: - """Rate-limit root-DM lobby reminders to one per cooldown window, not one per prompt typed.""" - if not hasattr(self, "_telegram_lobby_reminder_ts"): - self._telegram_lobby_reminder_ts = {} + def _telegram_cooldown_elapsed(self, source: SessionSource, attr: str, cooldown_s: float) -> bool: + """Per-(profile, chat) debounce: True (and stamp now) when the window has elapsed.""" + if not hasattr(self, attr): + setattr(self, attr, {}) key = self._telegram_topic_cooldown_key(source) if not key: return True - import time as _time - now = _time.monotonic() - last = self._telegram_lobby_reminder_ts.get(key, 0.0) - if now - last < self._TELEGRAM_LOBBY_REMINDER_COOLDOWN_S: + stamps = getattr(self, attr) + now = time.monotonic() + if now - stamps.get(key, 0.0) < cooldown_s: return False - self._telegram_lobby_reminder_ts[key] = now + stamps[key] = now return True + def _should_send_telegram_lobby_reminder(self, source: SessionSource) -> bool: + """Rate-limit root-DM lobby reminders to one per cooldown window, not one per prompt typed.""" + return self._telegram_cooldown_elapsed( + source, "_telegram_lobby_reminder_ts", self._TELEGRAM_LOBBY_REMINDER_COOLDOWN_S, + ) + + def _should_send_telegram_capability_hint(self, source: SessionSource) -> bool: + """Rate-limit the BotFather Threads Settings screenshot: repeated /topic while Threads + Settings are still off must not re-upload it every time.""" + return self._telegram_cooldown_elapsed( + source, "_telegram_capability_hint_ts", self._TELEGRAM_CAPABILITY_HINT_COOLDOWN_S, + ) + + # ── Telegram topic mode: user-facing text ─────────────────────────────────────────────── + def _telegram_topic_root_lobby_message(self) -> str: return ( "This main chat is reserved for system commands.\n\n" @@ -134,578 +155,6 @@ class GatewayTopicThreadsMixin: "the session attached to the current topic." ) - def _record_telegram_topic_binding( - self, - source: SessionSource, - session_entry, - ) -> None: - """Persist the Telegram topic -> Hermes session binding for topic lanes.""" - session_db = getattr(self, "_session_db", None) - if session_db is None or not source.chat_id or not source.thread_id: - return - # Runs off-loop (always via asyncio.to_thread); use the sync handle. - session_db = getattr(session_db, "_db", session_db) - session_db.bind_telegram_topic( - chat_id=str(source.chat_id), - thread_id=str(source.thread_id), - user_id=str(source.user_id or ""), - session_key=session_entry.session_key, - session_id=session_entry.session_id, - profile_name=self._telegram_topic_profile_name(source), - ) - - def _sync_telegram_topic_binding( - self, - source: SessionSource, - session_entry, - *, - reason: str, - ) -> None: - """Update the topic binding to point at ``session_entry.session_id``. - - Topic lanes persist (chat_id, thread_id) -> session_id so reopening a topic resumes the - right session. When compression rotates the id mid-turn a stale binding reloads the - oversized parent next message, retriggering preflight compression — sometimes in a loop. - """ - if not self._is_telegram_topic_lane(source): - return - try: - self._record_telegram_topic_binding(source, session_entry) - except Exception: - logger.debug( - "telegram topic binding refresh failed (%s)", reason, exc_info=True, - ) - - def _recover_telegram_topic_thread_id( - self, - source: SessionSource, - ) -> Optional[str]: - """Pin DM-topic routing to the user's last-active topic. - - Telegram can omit ``message_thread_id`` or surface General (``1``) for topic-mode DM - replies; in those lobby-shaped cases keep the conversation on the user's most-recent bound - topic. Do not rewrite a non-lobby, previously-unbound thread id: a brand-new DM topic is - also "unknown" until its first inbound message is recorded, and rewriting would send its - answer into an older lane. Returns None to leave the source alone. - """ - if ( - source.platform != Platform.TELEGRAM - or source.chat_type != "dm" - or not source.chat_id - or not source.user_id - or not self._telegram_topic_mode_enabled(source) - ): - return None - inbound = str(source.thread_id or "") - is_lobby = not inbound or inbound in self._TELEGRAM_GENERAL_TOPIC_IDS - if not is_lobby: - # A non-lobby, unknown thread_id is likely the first message of a new Telegram DM topic: - # preserve it to be recorded as a new lane below rather than hijack the latest binding. - return None - session_db = getattr(self, "_session_db", None) - if session_db is None: - return None - # Runs off-loop (always via asyncio.to_thread); use the sync handle. - session_db = getattr(session_db, "_db", session_db) - try: - bindings = session_db.list_telegram_topic_bindings_for_chat( - chat_id=str(source.chat_id), - profile_name=self._telegram_topic_profile_name(source), - ) - except Exception: - logger.debug("topic-recover: read failed", exc_info=True) - return None - if not bindings: - return None - user_id = str(source.user_id) - for b in bindings: # newest-first - if str(b.get("user_id") or "") == user_id: - recovered = str(b.get("thread_id") or "") - if recovered and recovered != inbound: - return recovered - return None - return None - - async def _get_telegram_topic_capabilities(self, source: SessionSource) -> dict: - """Read Telegram private-topic capability flags via Bot API getMe.""" - adapter = self._adapter_for_source(source) - bot = getattr(adapter, "_bot", None) - if bot is None or not hasattr(bot, "get_me"): - return {"checked": False} - try: - me = await bot.get_me() - except Exception: - logger.debug("Failed to fetch Telegram getMe topic capabilities", exc_info=True) - return {"checked": False} - - def _field(name: str): - if hasattr(me, name): - return getattr(me, name) - api_kwargs = getattr(me, "api_kwargs", None) - if isinstance(api_kwargs, dict) and name in api_kwargs: - return api_kwargs.get(name) - if isinstance(me, dict): - return me.get(name) - return None - - return { - "checked": True, - "has_topics_enabled": _field("has_topics_enabled"), - "allows_users_to_create_topics": _field("allows_users_to_create_topics"), - } - - async def _ensure_telegram_system_topic(self, source: SessionSource) -> None: - """Create/pin the managed System topic after /topic activation when possible.""" - adapter = self._adapter_for_source(source) - if adapter is None or not source.chat_id: - return - - thread_id = None - create_topic = getattr(adapter, "_create_dm_topic", None) - if callable(create_topic): - try: - thread_id = await create_topic(int(source.chat_id), "System") - except Exception: - logger.debug("Failed to create Telegram System topic", exc_info=True) - if not thread_id: - return - - message_id = None - try: - send_result = await adapter.send( - source.chat_id, - "System topic for Hermes commands and status.", - metadata={"thread_id": str(thread_id)}, - ) - message_id = getattr(send_result, "message_id", None) - except Exception: - logger.debug("Failed to send Telegram System topic intro", exc_info=True) - if not message_id: - return - - bot = getattr(adapter, "_bot", None) - if bot is None or not hasattr(bot, "pin_chat_message"): - return - try: - await bot.pin_chat_message( - chat_id=int(source.chat_id), - message_id=int(message_id), - disable_notification=True, - ) - except Exception: - logger.debug("Failed to pin Telegram System topic intro", exc_info=True) - - async def _send_telegram_topic_setup_image(self, source: SessionSource) -> None: - """Send the bundled BotFather Threads Settings screenshot when available.""" - adapter = self._adapter_for_source(source) - if adapter is None or not source.chat_id or not hasattr(adapter, "send_image_file"): - return - image_path = Path(__file__).resolve().parent / "assets" / "telegram-botfather-threads-settings.jpg" - if not image_path.exists(): - return - try: - await adapter.send_image_file( - chat_id=source.chat_id, - image_path=str(image_path), - caption="BotFather → Bot Settings → Threads Settings", - metadata={"thread_id": str(source.thread_id)} if source.thread_id else None, - ) - except Exception: - logger.debug("Failed to send Telegram topic setup image", exc_info=True) - - def _sanitize_telegram_topic_title(self, title: str) -> str: - """Return a Bot API-safe forum topic name from a generated session title.""" - cleaned = re.sub(r"\s+", " ", str(title or "")).strip() - if not cleaned: - return "Hermes Chat" - # Telegram forum topic names are short (currently 1-128 chars). Keep - # extra room for multi-byte titles and avoid trailing ellipsis churn. - if len(cleaned) > 120: - cleaned = cleaned[:117].rstrip() + "..." - return cleaned - - def _is_discord_auto_thread_lane(self, source: SessionSource) -> bool: - """Return True only for Discord threads Hermes just auto-created.""" - return ( - source.platform == Platform.DISCORD - and source.chat_type == "thread" - and bool(getattr(source, "auto_thread_created", False)) - and bool(source.thread_id) - and bool(getattr(source, "auto_thread_initial_name", None)) - ) - - def _is_relay_discord_channel_lane(self, source: SessionSource) -> bool: - """Shape-only check: a relay-delivered Discord CHANNEL event whose - reply the connector MAY auto-thread (title-turn registration gate). - - Deliberately does NOT consult the send-result cache: at registration - time (before delivery) the feedback can't exist yet. The rename lane - polls the cache at fire time instead.""" - return ( - source.platform == Platform.DISCORD - and bool(source.chat_id) - and not source.thread_id - and source.chat_type in ("group", "channel") - and getattr(source, "delivered_via_upstream_relay", False) is True - ) - - def _relay_auto_thread_info( - self, source: SessionSource - ) -> Optional[Tuple[str, str]]: - """(thread_id, initial_name) when the RELAY connector auto-threaded our reply to this - source's chat — the title-turn sibling of _is_discord_auto_thread_lane. - - The marker check only matches events ARRIVING IN an auto-created thread (turn 2+); the - auto-title fires on the FIRST exchange, whose source is the PARENT channel event with no - markers. Preferred: the connector's ``prospective_thread_id`` stamp (anchor message id == - the thread it will create) — per-message, so it names the EXACT thread even when several - auto-threads spawn from one channel; the connector's created-name guard enforces - no-clobber. Fallback: the per-chat send-result thread_id/auto_thread_name cache (older - connectors), which only ever renamed the FIRST thread. - """ - from gateway.run import _as_thread_info - if source.platform != Platform.DISCORD or not source.chat_id: - return None - if not getattr(source, "delivered_via_upstream_relay", False): - return None - prospective = getattr(source, "prospective_thread_id", None) - if prospective: - # Deterministic per-thread identity; the empty initial-name marker - # signals the caller to rely on the connector-side no-clobber guard. - return (str(prospective), "") - adapter = self._adapter_for_source(source) - info_fn = getattr(adapter, "auto_thread_info_for_chat", None) - if not callable(info_fn): - return None - try: - return _as_thread_info(info_fn(str(source.chat_id))) - except Exception: - return None - - async def _await_relay_auto_thread_info( - self, source: SessionSource - ) -> Optional[Tuple[str, str]]: - """``_relay_auto_thread_info``, waited out until this turn delivers. - - The legacy send-result path can only answer once the reply is sent, and the caller asks - at title time — one turn early. The adapter answers on the send either way, so the - timeout is only a backstop for a turn that never sends at all; the turn's own inactivity - limit is exactly how long that turn could still be alive. - """ - from gateway.run import _as_thread_info, _float_env - # The connector-stamped prospective id is known at ingest, so most - # sessions answer here and never wait at all. - known = self._relay_auto_thread_info(source) - if known is not None: - return known - adapter = self._adapter_for_source(source) - wait_fn = getattr(adapter, "wait_for_auto_thread_info", None) - if not callable(wait_fn) or not source.chat_id: - return None - # 0 means the operator disabled the turn limit; the backstop still needs one. - timeout = _float_env("HERMES_AGENT_TIMEOUT", 1800) or 1800 - try: - return _as_thread_info(await wait_fn(str(source.chat_id), timeout)) - except Exception: - return None - - def _sanitize_discord_thread_title(self, title: str) -> str: - """Return a Discord-safe semantic thread title from a session title. - - Discord thread names are capped at 100 characters measured in UTF-16 code units (emoji - count double), so truncate with the UTF-16 helpers rather than Python code-point slices. - """ - cleaned = re.sub(r"\s+", " ", str(title or "")).strip() - if not cleaned: - return "Hermes Chat" - if utf16_len(cleaned) > 80: - cleaned = _prefix_within_utf16_limit(cleaned, 77).rstrip() + "..." - return cleaned - - async def _rename_discord_auto_thread_for_session_title( - self, - source: SessionSource, - session_id: str, - title: str, - relay_info: Optional[Tuple[str, str]] = None, - ) -> None: - """Best-effort semantic rename of a newly auto-created Discord thread. - - ``relay_info`` is the (thread_id, initial_name) pair from the relay connector's send- - result feedback — supplied on the title turn, where the source is the parent-channel - event and carries no auto-thread markers (see _relay_auto_thread_info). - """ - if relay_info is None and not await asyncio.to_thread( - self._is_discord_auto_thread_lane, source - ): - # Relay title turn with no feedback captured at schedule time: the title comes off the - # user's opening message, so it beats the delivery that produces the connector's send- - # result feedback (thread_id + initial name) by the whole length of the turn. - if not self._is_relay_discord_channel_lane(source): - return - relay_info = await self._await_relay_auto_thread_info(source) - if relay_info is None: - # True miss: the connector did not auto-thread this reply - # (policy off, DM, already-threaded, or send failed). - return - adapter = self._adapter_for_source(source) if getattr(self, "adapters", None) else None - if adapter is None: - return - rename_thread = getattr(adapter, "rename_thread", None) - if rename_thread is None: - return - target_thread_id = relay_info[0] if relay_info else str(source.thread_id) - # Relay lane (relay_info present): ask the CONNECTOR to enforce the no-clobber guard from - # its own created-name memory — the gateway can't reliably reproduce the thread's initial - # name byte-for-byte (normalization drift silently declined every rename before this). - use_connector_guard = relay_info is not None - guard_name = ( - None - if use_connector_guard - else getattr(source, "auto_thread_initial_name", None) - ) - thread_name = self._sanitize_discord_thread_title(title) - # Relay lane only: the connector's egress guard resolves the owning tenant from the - # outbound scope_id/user_id caches, keyed by the PARENT channel chat_id (learned at - # inbound), not the thread id. rename_thread defaults chat_id to the thread id, so the - # lookup misses and the connector declines; pass the parent channel id (the relay source's - # chat_id). Native lane needs nothing: its source IS the thread, direct Discord API. - parent_chat_id = ( - str(source.chat_id) if use_connector_guard and source.chat_id else None - ) - logger.info( - "discord auto-thread rename: thread=%s lane=%s new_title=%r", - target_thread_id, - "relay" if use_connector_guard else "native", - thread_name, - ) - rename_kwargs = ( - { - "prefer_connector_created": True, - "parent_chat_id": parent_chat_id, - } - if use_connector_guard - else {"only_if_current_name": guard_name} - ) - try: - renamed = await rename_thread( - target_thread_id, - thread_name, - **rename_kwargs, - ) - logger.info( - "discord auto-thread rename result: thread=%s applied=%s", - target_thread_id, - bool(renamed), - ) - except TypeError: - logger.warning( - "Discord semantic thread rename raised TypeError (adapter=%s)", - type(adapter).__name__, - exc_info=True, - ) - except Exception: - logger.debug("Failed to rename Discord auto-thread for generated session title", exc_info=True) - - def _schedule_rename_from_title_thread(self, source: SessionSource, make_coro, label: str) -> None: - """Schedule a best-effort rename coroutine onto the gateway loop from the auto-title thread. - - The source is copied so the background thread never shares the live dataclass with the - loop; failures are logged at debug and never propagate.""" - from gateway.run import safe_schedule_threadsafe - try: - loop = asyncio.get_running_loop() - except RuntimeError: - loop = getattr(self, "_gateway_loop", None) - if loop is None or loop.is_closed(): - return - try: - copied_source = dataclasses.replace(source) - except Exception: - copied_source = source - future = safe_schedule_threadsafe( - make_coro(copied_source), - loop, - logger=logger, - log_message=f"{label} failed to schedule", - ) - if future is None: - return - - def _log_rename_failure(fut) -> None: - try: - fut.result() - except Exception: - logger.debug("%s failed", label, exc_info=True) - - future.add_done_callback(_log_rename_failure) - - def _schedule_discord_semantic_thread_rename( - self, - source: SessionSource, - session_id: str, - title: str, - ) -> None: - """Schedule Discord auto-thread rename from the auto-title background thread.""" - relay_info = None - if not title: - return - if not self._is_discord_auto_thread_lane(source): - # Relay title turn: the source is the PARENT channel event (thread didn't exist at - # ingest, no auto-thread markers). The connector's send-result feedback says where the - # reply landed, but the auto-title races that delivery, so a cache miss HERE is not a - # verdict. Schedule whenever the SHAPE matches; the async rename lane polls the cache - # (bounded wait) and no-ops on a true miss. - relay_info = self._relay_auto_thread_info(source) - if relay_info is None and not self._is_relay_discord_channel_lane( - source - ): - return - self._schedule_rename_from_title_thread( - source, - lambda copied: self._rename_discord_auto_thread_for_session_title( - copied, session_id, title, relay_info=relay_info - ), - "Discord semantic thread rename", - ) - - async def _rename_telegram_topic_for_session_title( - self, - source: SessionSource, - session_id: str, - title: str, - ) -> None: - """Best-effort rename of a Telegram DM topic when Hermes auto-titles a session.""" - if not await asyncio.to_thread(self._is_telegram_topic_lane, source) or not source.chat_id or not source.thread_id: - return - - # extra.disable_topic_auto_rename lets the operator disable per-topic auto-rename entirely, - # e.g. user-managed topics (ad-hoc Threaded Mode) that auto-rename would keep overwriting. - if self._telegram_topic_auto_rename_disabled(source): - return - - # Skip rename when the topic is operator-declared via extra.dm_topics. Those topics have - # fixed names chosen by the operator (plus optional skill binding); auto-renaming would - # silently mutate operator config. Check the class, not the instance — getattr() on a - # MagicMock auto-creates attributes, so an instance hasattr() is True for every test double. - adapter = self._adapter_for_source(source) - if adapter is not None: - get_info = getattr(type(adapter), "_get_dm_topic_info", None) - if callable(get_info): - try: - operator_topic = get_info(adapter, str(source.chat_id), str(source.thread_id)) - except Exception: - operator_topic = None - # Only treat dict-shaped returns as operator-declared; a - # bare MagicMock or other sentinel shouldn't count. - if isinstance(operator_topic, dict): - return - - session_db = getattr(self, "_session_db", None) - if session_db is not None: - try: - binding = await session_db.get_telegram_topic_binding( - chat_id=str(source.chat_id), - thread_id=str(source.thread_id), - profile_name=self._telegram_topic_profile_name(source), - ) - if binding and str(binding.get("session_id") or "") != str(session_id): - return - except Exception: - logger.debug("Failed to verify Telegram topic binding before rename", exc_info=True) - return - - if adapter is None: - return - topic_name = self._sanitize_telegram_topic_title(title) - try: - rename_topic = getattr(adapter, "rename_dm_topic", None) - if rename_topic is not None: - await rename_topic( - chat_id=str(source.chat_id), - thread_id=str(source.thread_id), - name=topic_name, - ) - return - - bot = getattr(adapter, "_bot", None) - edit_forum_topic = getattr(bot, "edit_forum_topic", None) if bot is not None else None - if edit_forum_topic is None: - edit_forum_topic = getattr(bot, "editForumTopic", None) if bot is not None else None - if edit_forum_topic is None: - return - try: - await edit_forum_topic( - chat_id=int(source.chat_id), - message_thread_id=int(source.thread_id), - name=topic_name, - ) - except (TypeError, ValueError): - await edit_forum_topic( - chat_id=source.chat_id, - message_thread_id=source.thread_id, - name=topic_name, - ) - except Exception: - logger.debug("Failed to rename Telegram topic for auto-generated title", exc_info=True) - - def _telegram_topic_auto_rename_disabled(self, source: SessionSource) -> bool: - """Return True when operator disabled per-topic auto-rename for this Telegram chat. - - ``gateway.platforms.telegram.extra.disable_topic_auto_rename``; default False (auto-rename on). - """ - platform_cfg = ( - self.config.platforms.get(source.platform) - if getattr(self, "config", None) and getattr(self.config, "platforms", None) - else None - ) - if platform_cfg is None: - return False - extra = getattr(platform_cfg, "extra", None) or {} - value = extra.get("disable_topic_auto_rename") - if value is None: - return False - if isinstance(value, bool): - return value - if isinstance(value, str): - return value.strip().lower() in {"1", "true", "yes", "on"} - return bool(value) - - def _schedule_telegram_topic_title_rename( - self, - source: SessionSource, - session_id: str, - title: str, - ) -> None: - """Schedule a topic rename from the auto-title background thread.""" - if not title or not self._is_telegram_topic_lane(source): - return - if self._telegram_topic_auto_rename_disabled(source): - return - self._schedule_rename_from_title_thread( - source, - lambda copied: self._rename_telegram_topic_for_session_title(copied, session_id, title), - "Telegram topic title rename", - ) - - def _should_send_telegram_capability_hint(self, source: SessionSource) -> bool: - """Rate-limit the BotFather Threads Settings screenshot. - - Repeated /topic while Threads Settings are still off must not re-upload it every time. - """ - if not hasattr(self, "_telegram_capability_hint_ts"): - self._telegram_capability_hint_ts = {} - key = self._telegram_topic_cooldown_key(source) - if not key: - return True - import time as _time - now = _time.monotonic() - last = self._telegram_capability_hint_ts.get(key, 0.0) - if now - last < self._TELEGRAM_CAPABILITY_HINT_COOLDOWN_S: - return False - self._telegram_capability_hint_ts[key] = now - return True - def _telegram_topic_help_text(self) -> str: return ( "/topic — enable multi-session DM mode (one bot, many parallel chats)\n" @@ -728,6 +177,444 @@ class GatewayTopicThreadsMixin: "5. /topic inside a topic restores an old session into it." ) + # ── Telegram topic bindings ───────────────────────────────────────────────────────────── + + def _record_telegram_topic_binding(self, source: SessionSource, session_entry) -> None: + """Persist the Telegram topic -> Hermes session binding for topic lanes (off-loop).""" + session_db = self._sync_session_db() + if session_db is None or not source.chat_id or not source.thread_id: + return + session_db.bind_telegram_topic( + chat_id=str(source.chat_id), thread_id=str(source.thread_id), + user_id=str(source.user_id or ""), session_key=session_entry.session_key, + session_id=session_entry.session_id, + profile_name=self._telegram_topic_profile_name(source), + ) + + def _sync_telegram_topic_binding(self, source: SessionSource, session_entry, *, reason: str) -> None: + """Update the topic binding to point at ``session_entry.session_id``. + + Topic lanes persist (chat_id, thread_id) -> session_id so reopening a topic resumes the + right session. When compression rotates the id mid-turn a stale binding reloads the + oversized parent next message, retriggering preflight compression — sometimes in a loop. + """ + if not self._is_telegram_topic_lane(source): + return + try: + self._record_telegram_topic_binding(source, session_entry) + except Exception: + logger.debug("telegram topic binding refresh failed (%s)", reason, exc_info=True) + + def _recover_telegram_topic_thread_id(self, source: SessionSource) -> Optional[str]: + """Pin DM-topic routing to the user's last-active topic. + + Telegram can omit ``message_thread_id`` or surface General (``1``) for topic-mode DM + replies; in those lobby-shaped cases keep the conversation on the user's most-recent bound + topic. Do not rewrite a non-lobby, previously-unbound thread id: a brand-new DM topic is + also "unknown" until its first inbound message is recorded, and rewriting would send its + answer into an older lane. Returns None to leave the source alone. + """ + if ( + not self._is_telegram_dm(source) + or not source.chat_id + or not source.user_id + or not self._telegram_topic_mode_enabled(source) + ): + return None + inbound = str(source.thread_id or "") + if inbound and inbound not in self._TELEGRAM_GENERAL_TOPIC_IDS: + return None + session_db = self._sync_session_db() + if session_db is None: + return None + try: + bindings = session_db.list_telegram_topic_bindings_for_chat( + chat_id=str(source.chat_id), profile_name=self._telegram_topic_profile_name(source) + ) + except Exception: + logger.debug("topic-recover: read failed", exc_info=True) + return None + user_id = str(source.user_id) + for b in bindings or (): # newest-first + if str(b.get("user_id") or "") == user_id: + recovered = str(b.get("thread_id") or "") + return recovered if recovered and recovered != inbound else None + return None + + # ── Telegram topic mode: /topic activation helpers ────────────────────────────────────── + + async def _get_telegram_topic_capabilities(self, source: SessionSource) -> dict: + """Read Telegram private-topic capability flags via Bot API getMe.""" + bot = getattr(self._adapter_for_source(source), "_bot", None) + if bot is None or not hasattr(bot, "get_me"): + return {"checked": False} + try: + me = await bot.get_me() + except Exception: + logger.debug("Failed to fetch Telegram getMe topic capabilities", exc_info=True) + return {"checked": False} + + def _field(name: str): + if hasattr(me, name): + return getattr(me, name) + api_kwargs = getattr(me, "api_kwargs", None) + if isinstance(api_kwargs, dict) and name in api_kwargs: + return api_kwargs.get(name) + return me.get(name) if isinstance(me, dict) else None + + return { + "checked": True, + "has_topics_enabled": _field("has_topics_enabled"), + "allows_users_to_create_topics": _field("allows_users_to_create_topics"), + } + + async def _ensure_telegram_system_topic(self, source: SessionSource) -> None: + """Create/pin the managed System topic after /topic activation when possible.""" + adapter = self._adapter_for_source(source) + if adapter is None or not source.chat_id: + return + thread_id = None + create_topic = getattr(adapter, "_create_dm_topic", None) + if callable(create_topic): + try: + thread_id = await create_topic(int(source.chat_id), "System") + except Exception: + logger.debug("Failed to create Telegram System topic", exc_info=True) + if not thread_id: + return + message_id = None + try: + send_result = await adapter.send( + source.chat_id, "System topic for Hermes commands and status.", + metadata={"thread_id": str(thread_id)}, + ) + message_id = getattr(send_result, "message_id", None) + except Exception: + logger.debug("Failed to send Telegram System topic intro", exc_info=True) + if not message_id: + return + bot = getattr(adapter, "_bot", None) + if bot is None or not hasattr(bot, "pin_chat_message"): + return + try: + await bot.pin_chat_message( + chat_id=int(source.chat_id), message_id=int(message_id), disable_notification=True + ) + except Exception: + logger.debug("Failed to pin Telegram System topic intro", exc_info=True) + + async def _send_telegram_topic_setup_image(self, source: SessionSource) -> None: + """Send the bundled BotFather Threads Settings screenshot when available.""" + adapter = self._adapter_for_source(source) + if adapter is None or not source.chat_id or not hasattr(adapter, "send_image_file"): + return + image_path = Path(__file__).resolve().parent / "assets" / "telegram-botfather-threads-settings.jpg" + if not image_path.exists(): + return + try: + await adapter.send_image_file( + chat_id=source.chat_id, image_path=str(image_path), + caption="BotFather → Bot Settings → Threads Settings", + metadata={"thread_id": str(source.thread_id)} if source.thread_id else None, + ) + except Exception: + logger.debug("Failed to send Telegram topic setup image", exc_info=True) + + # ── title sanitizers ──────────────────────────────────────────────────────────────────── + + def _sanitize_telegram_topic_title(self, title: str) -> str: + """Bot API-safe forum topic name: names are 1-128 chars; keep room for multi-byte titles.""" + cleaned = re.sub(r"\s+", " ", str(title or "")).strip() + if not cleaned: + return "Hermes Chat" + if len(cleaned) > 120: + cleaned = cleaned[:117].rstrip() + "..." + return cleaned + + def _sanitize_discord_thread_title(self, title: str) -> str: + """Discord-safe thread title: the 100-char cap is measured in UTF-16 code units (emoji count + double), so truncate with the UTF-16 helpers rather than Python code-point slices.""" + cleaned = re.sub(r"\s+", " ", str(title or "")).strip() + if not cleaned: + return "Hermes Chat" + if utf16_len(cleaned) > 80: + cleaned = _prefix_within_utf16_limit(cleaned, 77).rstrip() + "..." + return cleaned + + # ── Discord auto-thread lanes ─────────────────────────────────────────────────────────── + + def _is_discord_auto_thread_lane(self, source: SessionSource) -> bool: + """Return True only for Discord threads Hermes just auto-created.""" + return ( + source.platform == Platform.DISCORD + and source.chat_type == "thread" + and bool(getattr(source, "auto_thread_created", False)) + and bool(source.thread_id) + and bool(getattr(source, "auto_thread_initial_name", None)) + ) + + def _is_relay_discord_channel_lane(self, source: SessionSource) -> bool: + """Shape-only check: a relay-delivered Discord CHANNEL event whose reply the connector MAY + auto-thread (title-turn registration gate). + + Deliberately does NOT consult the send-result cache: at registration time (before delivery) + the feedback can't exist yet. The rename lane polls the cache at fire time instead.""" + return ( + source.platform == Platform.DISCORD + and bool(source.chat_id) + and not source.thread_id + and source.chat_type in ("group", "channel") + and getattr(source, "delivered_via_upstream_relay", False) is True + ) + + def _relay_auto_thread_info(self, source: SessionSource) -> Optional[Tuple[str, str]]: + """(thread_id, initial_name) when the RELAY connector auto-threaded our reply to this + source's chat — the title-turn sibling of _is_discord_auto_thread_lane. + + The marker check only matches events ARRIVING IN an auto-created thread (turn 2+); the + auto-title fires on the FIRST exchange, whose source is the PARENT channel event with no + markers. Preferred: the connector's ``prospective_thread_id`` stamp (anchor message id == + the thread it will create) — per-message, so it names the EXACT thread even when several + auto-threads spawn from one channel; the connector's created-name guard enforces + no-clobber. Fallback: the per-chat send-result thread_id/auto_thread_name cache (older + connectors), which only ever renamed the FIRST thread. + """ + from gateway.run import _as_thread_info + if source.platform != Platform.DISCORD or not source.chat_id: + return None + if not getattr(source, "delivered_via_upstream_relay", False): + return None + prospective = getattr(source, "prospective_thread_id", None) + if prospective: + # Deterministic per-thread identity; the empty initial-name marker signals the caller + # to rely on the connector-side no-clobber guard. + return (str(prospective), "") + info_fn = getattr(self._adapter_for_source(source), "auto_thread_info_for_chat", None) + if not callable(info_fn): + return None + try: + return _as_thread_info(info_fn(str(source.chat_id))) + except Exception: + return None + + async def _await_relay_auto_thread_info(self, source: SessionSource) -> Optional[Tuple[str, str]]: + """``_relay_auto_thread_info``, waited out until this turn delivers. + + The legacy send-result path can only answer once the reply is sent, and the caller asks + at title time — one turn early. The adapter answers on the send either way, so the + timeout is only a backstop for a turn that never sends at all; the turn's own inactivity + limit is exactly how long that turn could still be alive. + """ + from gateway.run import _as_thread_info, _float_env + # The connector-stamped prospective id is known at ingest, so most sessions answer here. + known = self._relay_auto_thread_info(source) + if known is not None: + return known + wait_fn = getattr(self._adapter_for_source(source), "wait_for_auto_thread_info", None) + if not callable(wait_fn) or not source.chat_id: + return None + # 0 means the operator disabled the turn limit; the backstop still needs one. + timeout = _float_env("HERMES_AGENT_TIMEOUT", 1800) or 1800 + try: + return _as_thread_info(await wait_fn(str(source.chat_id), timeout)) + except Exception: + return None + + async def _rename_discord_auto_thread_for_session_title( + self, source: SessionSource, session_id: str, title: str, + relay_info: Optional[Tuple[str, str]] = None, + ) -> None: + """Best-effort semantic rename of a newly auto-created Discord thread. + + ``relay_info`` is the (thread_id, initial_name) pair from the relay connector's send- + result feedback — supplied on the title turn, where the source is the parent-channel + event and carries no auto-thread markers (see _relay_auto_thread_info). + """ + if relay_info is None and not await asyncio.to_thread(self._is_discord_auto_thread_lane, source): + # Relay title turn with no feedback captured at schedule time: the title comes off the + # user's opening message, so it beats the delivery that produces the connector's + # send-result feedback by the whole length of the turn. + if not self._is_relay_discord_channel_lane(source): + return + relay_info = await self._await_relay_auto_thread_info(source) + if relay_info is None: + # True miss: the connector did not auto-thread this reply. + return + adapter = self._adapter_for_source(source) if getattr(self, "adapters", None) else None + rename_thread = getattr(adapter, "rename_thread", None) + if rename_thread is None: + return + relay = relay_info is not None + target_thread_id = relay_info[0] if relay else str(source.thread_id) + thread_name = self._sanitize_discord_thread_title(title) + if relay: + # Ask the CONNECTOR to enforce the no-clobber guard from its own created-name memory — + # the gateway can't reproduce the thread's initial name byte-for-byte (normalization + # drift silently declined every rename). The connector's egress guard resolves the + # owning tenant from caches keyed by the PARENT channel chat_id (learned at inbound), + # not the thread id, so pass the parent channel id or the lookup misses and it declines. + rename_kwargs = { + "prefer_connector_created": True, + "parent_chat_id": str(source.chat_id) if source.chat_id else None, + } + else: + # Native lane: the source IS the thread (direct Discord API); guard on the initial name. + rename_kwargs = {"only_if_current_name": getattr(source, "auto_thread_initial_name", None)} + logger.info( + "discord auto-thread rename: thread=%s lane=%s new_title=%r", + target_thread_id, "relay" if relay else "native", thread_name, + ) + try: + renamed = await rename_thread(target_thread_id, thread_name, **rename_kwargs) + logger.info( + "discord auto-thread rename result: thread=%s applied=%s", + target_thread_id, bool(renamed), + ) + except TypeError: + logger.warning( + "Discord semantic thread rename raised TypeError (adapter=%s)", + type(adapter).__name__, exc_info=True, + ) + except Exception: + logger.debug("Failed to rename Discord auto-thread for generated session title", exc_info=True) + + # ── title-thread → loop rename scheduling ─────────────────────────────────────────────── + + def _schedule_rename_from_title_thread(self, source: SessionSource, make_coro, label: str) -> None: + """Schedule a best-effort rename coroutine onto the gateway loop from the auto-title thread. + + The source is copied so the background thread never shares the live dataclass with the + loop; failures are logged at debug and never propagate.""" + from gateway.run import safe_schedule_threadsafe + try: + loop = asyncio.get_running_loop() + except RuntimeError: + loop = getattr(self, "_gateway_loop", None) + if loop is None or loop.is_closed(): + return + try: + copied_source = dataclasses.replace(source) + except Exception: + copied_source = source + future = safe_schedule_threadsafe( + make_coro(copied_source), loop, logger=logger, log_message=f"{label} failed to schedule", + ) + if future is None: + return + + def _log_rename_failure(fut) -> None: + try: + fut.result() + except Exception: + logger.debug("%s failed", label, exc_info=True) + + future.add_done_callback(_log_rename_failure) + + def _schedule_discord_semantic_thread_rename(self, source: SessionSource, session_id: str, title: str) -> None: + """Schedule Discord auto-thread rename from the auto-title background thread.""" + relay_info = None + if not title: + return + if not self._is_discord_auto_thread_lane(source): + # Relay title turn: the source is the PARENT channel event (thread didn't exist at + # ingest). The auto-title races the delivery that fills the send-result cache, so a + # miss HERE is not a verdict: schedule whenever the SHAPE matches; the async rename + # lane polls the cache (bounded wait) and no-ops on a true miss. + relay_info = self._relay_auto_thread_info(source) + if relay_info is None and not self._is_relay_discord_channel_lane(source): + return + self._schedule_rename_from_title_thread( + source, + lambda copied: self._rename_discord_auto_thread_for_session_title( + copied, session_id, title, relay_info=relay_info + ), + "Discord semantic thread rename", + ) + + def _schedule_telegram_topic_title_rename(self, source: SessionSource, session_id: str, title: str) -> None: + """Schedule a topic rename from the auto-title background thread.""" + if not title or not self._is_telegram_topic_lane(source): + return + if self._telegram_topic_auto_rename_disabled(source): + return + self._schedule_rename_from_title_thread( + source, + lambda copied: self._rename_telegram_topic_for_session_title(copied, session_id, title), + "Telegram topic title rename", + ) + + def _telegram_topic_auto_rename_disabled(self, source: SessionSource) -> bool: + """``gateway.platforms.telegram.extra.disable_topic_auto_rename``; default False (auto-rename on).""" + config = getattr(self, "config", None) + platform_cfg = config.platforms.get(source.platform) if config and getattr(config, "platforms", None) else None + if platform_cfg is None: + return False + value = (getattr(platform_cfg, "extra", None) or {}).get("disable_topic_auto_rename") + if value is None: + return False + if isinstance(value, str): + return value.strip().lower() in {"1", "true", "yes", "on"} + return bool(value) + + async def _rename_telegram_topic_for_session_title(self, source: SessionSource, session_id: str, title: str) -> None: + """Best-effort rename of a Telegram DM topic when Hermes auto-titles a session.""" + if not await asyncio.to_thread(self._is_telegram_topic_lane, source) or not source.chat_id or not source.thread_id: + return + # Operator kill-switch, e.g. user-managed topics (ad-hoc Threaded Mode) that auto-rename + # would keep overwriting. + if self._telegram_topic_auto_rename_disabled(source): + return + # Skip operator-declared topics (extra.dm_topics): fixed names chosen by the operator; + # auto-renaming would silently mutate operator config. Check the class, not the instance — + # getattr() on a MagicMock auto-creates attributes, so every test double would match. + adapter = self._adapter_for_source(source) + if adapter is not None: + get_info = getattr(type(adapter), "_get_dm_topic_info", None) + if callable(get_info): + try: + operator_topic = get_info(adapter, str(source.chat_id), str(source.thread_id)) + except Exception: + operator_topic = None + # Only dict-shaped returns count; a bare MagicMock or other sentinel shouldn't. + if isinstance(operator_topic, dict): + return + session_db = getattr(self, "_session_db", None) + if session_db is not None: + try: + binding = await session_db.get_telegram_topic_binding( + chat_id=str(source.chat_id), thread_id=str(source.thread_id), + profile_name=self._telegram_topic_profile_name(source), + ) + if binding and str(binding.get("session_id") or "") != str(session_id): + return + except Exception: + logger.debug("Failed to verify Telegram topic binding before rename", exc_info=True) + return + if adapter is None: + return + topic_name = self._sanitize_telegram_topic_title(title) + try: + rename_topic = getattr(adapter, "rename_dm_topic", None) + if rename_topic is not None: + await rename_topic(chat_id=str(source.chat_id), thread_id=str(source.thread_id), name=topic_name) + return + bot = getattr(adapter, "_bot", None) + edit_forum_topic = getattr(bot, "edit_forum_topic", None) or getattr(bot, "editForumTopic", None) + if edit_forum_topic is None: + return + try: + await edit_forum_topic( + chat_id=int(source.chat_id), message_thread_id=int(source.thread_id), name=topic_name, + ) + except (TypeError, ValueError): + await edit_forum_topic( + chat_id=source.chat_id, message_thread_id=source.thread_id, name=topic_name, + ) + except Exception: + logger.debug("Failed to rename Telegram topic for auto-generated title", exc_info=True) + + # ── /topic command bodies ─────────────────────────────────────────────────────────────── + async def _disable_telegram_topic_mode_for_chat(self, source: SessionSource) -> str: """Cleanly disable topic mode for a chat via /topic off.""" if not self._session_db: @@ -736,27 +623,21 @@ class GatewayTopicThreadsMixin: chat_id = str(source.chat_id or "") if not chat_id: return "Could not determine chat ID." - # No-op if never enabled. + profile_name = self._telegram_topic_profile_name(source) try: currently_enabled = await self._session_db.is_telegram_topic_mode_enabled( - chat_id=chat_id, - user_id=str(source.user_id or ""), - profile_name=self._telegram_topic_profile_name(source), + chat_id=chat_id, user_id=str(source.user_id or ""), profile_name=profile_name, ) except Exception: currently_enabled = False if not currently_enabled: return "Multi-session topic mode is not currently enabled for this chat." try: - await self._session_db.disable_telegram_topic_mode( - chat_id=chat_id, - profile_name=self._telegram_topic_profile_name(source), - ) + await self._session_db.disable_telegram_topic_mode(chat_id=chat_id, profile_name=profile_name) except Exception as exc: logger.exception("Failed to disable Telegram topic mode") return f"Failed to disable topic mode: {exc}" - # Reset per-profile+chat debounce state so the user doesn't see a - # stale cooldown on the next activation (issue #76423). + # Reset per-profile+chat debounce state so the next activation doesn't see a stale cooldown. cooldown_key = self._telegram_topic_cooldown_key(source) if cooldown_key: for attr in ("_telegram_lobby_reminder_ts", "_telegram_capability_hint_ts"): @@ -781,88 +662,67 @@ class GatewayTopicThreadsMixin: ] try: sessions = await self._session_db.list_unlinked_telegram_sessions_for_user( - chat_id=str(source.chat_id), - user_id=str(source.user_id), - profile_name=self._telegram_topic_profile_name(source), - limit=10, + chat_id=str(source.chat_id), user_id=str(source.user_id), + profile_name=self._telegram_topic_profile_name(source), limit=10, ) except Exception: logger.debug("Failed to list unlinked Telegram sessions", exc_info=True) sessions = [] - if sessions: lines.append("Previous unlinked sessions:") for session in sessions: - session_id = str(session.get("id") or "") - title = str(session.get("title") or "Untitled session") + line = f"- {session.get('title') or 'Untitled session'} — `{session.get('id') or ''}`" preview = str(session.get("preview") or "").strip() - line = f"- {title} — `{session_id}`" if preview: line += f" — {preview}" lines.append(line) lines.extend([ "", - "To restore one:", - "1. Create or open a topic. To create a new one, open All Messages and send any message there.", - "2. Send /topic inside that topic.", + "To restore one:", *_TOPIC_RESTORE_STEPS, f"Example: Send /topic {sessions[0].get('id')} inside a topic.", ]) else: lines.extend([ "No previous unlinked Telegram sessions found.", "", - "To restore a previous session later:", - "1. Create or open a topic. To create a new one, open All Messages and send any message there.", - "2. Send /topic inside that topic.", + "To restore a previous session later:", *_TOPIC_RESTORE_STEPS, ]) return "\n".join(lines) async def _restore_telegram_topic_session(self, event: MessageEvent, raw_session_id: str) -> str: """Restore an existing Telegram-owned Hermes session into this topic.""" source = event.source - session_id = await self._session_db.resolve_session_id(raw_session_id.strip()) - if not session_id: - return f"Session not found: {raw_session_id.strip()}" - - session = await self._session_db.get_session(session_id) + db = self._session_db + session_id = await db.resolve_session_id(raw_session_id.strip()) + session = await db.get_session(session_id) if session_id else None if not session: return f"Session not found: {raw_session_id.strip()}" if str(session.get("source") or "") != "telegram": return "That session is not a Telegram session and cannot be restored into this topic." if str(session.get("user_id") or "") != str(source.user_id): return "That session does not belong to this Telegram user." - - linked = await self._session_db.is_telegram_session_linked_to_topic(session_id=session_id) + linked = await db.is_telegram_session_linked_to_topic(session_id=session_id) topic_profile = self._telegram_topic_profile_name(source) - current_binding = await self._session_db.get_telegram_topic_binding( - chat_id=str(source.chat_id), - thread_id=str(source.thread_id), - profile_name=topic_profile, + current_binding = await db.get_telegram_topic_binding( + chat_id=str(source.chat_id), thread_id=str(source.thread_id), profile_name=topic_profile, ) - if linked: - if not current_binding or current_binding.get("session_id") != session_id: - return "That session is already linked to another Telegram topic." - - session_key = self._session_key_for_source(source) + already_linked = "That session is already linked to another Telegram topic." + if linked and (not current_binding or current_binding.get("session_id") != session_id): + return already_linked try: - await self._session_db.bind_telegram_topic( - chat_id=str(source.chat_id), - thread_id=str(source.thread_id), - user_id=str(source.user_id), - session_key=session_key, - session_id=session_id, - managed_mode="restored", - profile_name=topic_profile, + await db.bind_telegram_topic( + chat_id=str(source.chat_id), thread_id=str(source.thread_id), + user_id=str(source.user_id), session_key=self._session_key_for_source(source), + session_id=session_id, managed_mode="restored", profile_name=topic_profile, ) except ValueError as exc: if "already linked" in str(exc): - return "That session is already linked to another Telegram topic." + return already_linked raise - - title = await self._session_db.get_session_title(session_id) or session_id + title = await db.get_session_title(session_id) or session_id last_assistant = None try: - for message in reversed(await self._session_db.get_messages(session_id)): + for message in reversed(await db.get_messages(session_id)): if message.get("role") != "assistant": continue projected = project_compaction_message_for_display(message) @@ -871,7 +731,6 @@ class GatewayTopicThreadsMixin: break except Exception: last_assistant = None - response = f"Session restored: {title}" if last_assistant: response += f"\n\nLast Hermes message:\n{last_assistant}" diff --git a/gateway/run_turn.py b/gateway/run_turn.py index abdc10fa12..6befcc70a1 100644 --- a/gateway/run_turn.py +++ b/gateway/run_turn.py @@ -18,7 +18,7 @@ import queue import threading import time from agent.i18n import t -from contextlib import suppress +from contextlib import nullcontext, suppress from contextvars import copy_context from gateway.config import Platform from gateway.media_repair import repair_explicit_computer_use_media_paths @@ -48,10 +48,7 @@ class GatewayTurnMixin: """Agent-turn execution (_handle_message_with_agent, _run_agent*, proxy path, background tasks, MCP reload) for GatewayRunner.""" def _resolve_session_agent_runtime( - self, - *, - source: Optional[SessionSource] = None, - session_key: Optional[str] = None, + self, *, source: Optional[SessionSource] = None, session_key: Optional[str] = None, user_config: Optional[dict] = None, ) -> tuple[str, dict]: """Resolve model/runtime for a session. @@ -60,38 +57,25 @@ class GatewayTurnMixin: (``_resolve_gateway_model(user_config)`` and default provider resolution). """ from gateway.run import ( - _credential_pool_for_provider, - _get_channel_override, - _resolve_gateway_model, - _resolve_runtime_agent_kwargs, - _resolve_runtime_agent_kwargs_for_provider, + _credential_pool_for_provider, _get_channel_override, _resolve_gateway_model, + _resolve_runtime_agent_kwargs, _resolve_runtime_agent_kwargs_for_provider, ) - resolved_session_key = self._resolve_session_key_or_none(source, session_key) + skey = self._resolve_session_key_or_none(source, session_key) model = _resolve_gateway_model(user_config) - if resolved_session_key: - self._rehydrate_session_model_override(resolved_session_key) - _override_state = ( - self._peek_session_state(resolved_session_key) - if resolved_session_key - else None - ) - override = ( - _override_state.conversation.model_override if _override_state else None - ) + if skey: + self._rehydrate_session_model_override(skey) + _override_state = self._peek_session_state(skey) if skey else None + override = _override_state.conversation.model_override if _override_state else None if override: override_model = override.get("model", model) override_runtime = { - "provider": override.get("provider"), - "requested_provider": override.get("requested_provider"), - "api_key": override.get("api_key"), - "base_url": override.get("base_url"), - "api_mode": override.get("api_mode"), - "max_tokens": override.get("max_tokens"), - "credential_pool": override.get("credential_pool"), - "request_overrides": override.get("request_overrides"), - "capabilities": dict(override.get("capabilities") or {}), + k: override.get(k) for k in ( + "provider", "requested_provider", "api_key", "base_url", "api_mode", + "max_tokens", "credential_pool", "request_overrides", "capabilities", + ) } + override_runtime["capabilities"] = dict(override_runtime["capabilities"] or {}) if override_runtime.get("api_key"): if override_runtime.get("credential_pool") is None: override_runtime["credential_pool"] = _credential_pool_for_provider( @@ -99,23 +83,21 @@ class GatewayTurnMixin: ) logger.debug( "Session model override (fast): session=%s config_model=%s -> override_model=%s provider=%s", - resolved_session_key or "", model, override_model, - override_runtime.get("provider"), + skey or "", model, override_model, override_runtime.get("provider"), ) return override_model, override_runtime - # Override exists but has no api_key — fall through to env-based - # resolution and apply model/provider from the override on top. + # No api_key on the override: fall through to env-based resolution and apply the + # override's model/provider on top. logger.debug( "Session model override (no api_key, fallback): session=%s config_model=%s override_model=%s", - resolved_session_key or "", model, override_model, + skey or "", model, override_model, ) else: logger.debug( "No session model override: session=%s config_model=%s override_keys=%s", - resolved_session_key or "", model, + skey or "", model, [ - _key - for _key, _st in list(self._sessions_map().items()) + _key for _key, _st in list(self._sessions_map().items()) if _st.conversation.model_override is not None ][:5] or "[]", ) @@ -123,48 +105,28 @@ class GatewayTurnMixin: runtime_kwargs = _resolve_runtime_agent_kwargs() runtime_model = runtime_kwargs.pop("model", None) if runtime_model: - logger.info( - "Runtime provider supplied explicit model override: %s -> %s", - model, - runtime_model, - ) + logger.info("Runtime provider supplied explicit model override: %s -> %s", model, runtime_model) model = runtime_model - cfg = getattr(self, "config", None) + cfg = getattr(self, "config", None) # getattr: bare object.__new__ test runners if cfg and source is not None: - chat_id = str(source.chat_id) if source.chat_id else "" - thread_id = ( - str(source.thread_id) if getattr(source, "thread_id", None) else None - ) - parent_id = ( - str(source.parent_chat_id) - if getattr(source, "parent_chat_id", None) - else None - ) ch = _get_channel_override( - cfg, - source.platform, - chat_id, - thread_id=thread_id, - parent_id=parent_id, + cfg, source.platform, str(source.chat_id) if source.chat_id else "", + thread_id=str(source.thread_id) if getattr(source, "thread_id", None) else None, + parent_id=str(source.parent_chat_id) if getattr(source, "parent_chat_id", None) else None, ) if ch: if ch.model: model = ch.model if ch.provider: - runtime_kwargs = _resolve_runtime_agent_kwargs_for_provider( - ch.provider - ) + runtime_kwargs = _resolve_runtime_agent_kwargs_for_provider(ch.provider) ch_runtime_model = runtime_kwargs.pop("model", None) - # Only adopt the provider's bundled model when the override - # did not specify an explicit model. + # Adopt the provider's bundled model only when the override named none. if ch_runtime_model and not ch.model: model = ch_runtime_model - if override and resolved_session_key: - model, runtime_kwargs = self._apply_session_model_override( - resolved_session_key, model, runtime_kwargs - ) + if override and skey: + model, runtime_kwargs = self._apply_session_model_override(skey, model, runtime_kwargs) # No model.default but a provider resolved (e.g. `hermes auth add openai-codex` without # `hermes model`): fall back to the provider's first catalog model so the API call has one. @@ -180,16 +142,11 @@ class GatewayTurnMixin: except Exception: pass - # Final safety net: if resolution still produced an empty model (e.g. a transient config-cache - # miss on a post-interrupt recovery turn), reuse the last model resolved for this session, - # else the most recent process-wide — model="" makes every API call fail HTTP 400 and the - # session goes silent. ``getattr`` guards bare test runners built via ``object.__new__``. + # Final safety net: an empty model (e.g. transient config-cache miss on a post-interrupt + # recovery turn) makes every API call fail HTTP 400 and the session goes silent — reuse the + # last model resolved for this session, else the most recent process-wide. if not model: - _lr_state = ( - self._peek_session_state(resolved_session_key) - if resolved_session_key - else None - ) + _lr_state = self._peek_session_state(skey) if skey else None _lr_star = self._peek_session_state("*") _recovered = ( (_lr_state.conversation.last_resolved_model if _lr_state else "") @@ -200,15 +157,13 @@ class GatewayTurnMixin: "Empty model resolved for session=%s — recovering " "last-known-good model %s (config read likely returned " "empty; see #35314)", - resolved_session_key or "", _recovered, + skey or "", _recovered, ) model = _recovered - elif model: + else: # Cache the good resolution for future recovery turns. - if resolved_session_key: - self._session_state( - resolved_session_key - ).conversation.last_resolved_model = model + if skey: + self._session_state(skey).conversation.last_resolved_model = model self._session_state("*").conversation.last_resolved_model = model return model, runtime_kwargs @@ -216,74 +171,50 @@ class GatewayTurnMixin: def _resolve_turn_agent_config(self, user_message: str, model: str, runtime_kwargs: dict) -> dict: """Build the effective model/runtime config for a single turn. - Always uses the session's primary model/provider. If `/fast` is enabled and the model - supports it, attach `request_overrides` for priority processing. Per-provider - ``request_overrides`` from ``resolve_runtime_provider`` (e.g. ``custom_providers`` - ``extra_body``) are merged *under* the fast-mode overrides so they still reach the model. + Always uses the session's primary model/provider. With `/fast` priority on and a model that + supports it, fast-mode ``request_overrides`` are deep-merged OVER the per-provider + ``request_overrides`` (e.g. ``custom_providers`` ``extra_body``) so both reach the model. """ from gateway.run import _deep_merge_request_overrides from hermes_cli.models import resolve_fast_mode_overrides + # Tests bind this method onto bare namespaces, so no class-level tables here. runtime = { - "api_key": runtime_kwargs.get("api_key"), - "base_url": runtime_kwargs.get("base_url"), - "provider": runtime_kwargs.get("provider"), - "requested_provider": runtime_kwargs.get("requested_provider"), - "api_mode": runtime_kwargs.get("api_mode"), - "command": runtime_kwargs.get("command"), - "args": list(runtime_kwargs.get("args") or []), - "credential_pool": runtime_kwargs.get("credential_pool"), - "max_tokens": runtime_kwargs.get("max_tokens"), - "capabilities": dict(runtime_kwargs.get("capabilities") or {}), + k: runtime_kwargs.get(k) for k in ( + "api_key", "base_url", "provider", "requested_provider", "api_mode", "command", "args", + "credential_pool", "max_tokens", "capabilities", + ) } + runtime["args"] = list(runtime["args"] or []) + runtime["capabilities"] = dict(runtime["capabilities"] or {}) base_request_overrides = dict(runtime_kwargs.get("request_overrides") or {}) route = { "model": model, "runtime": runtime, "signature": ( - model, - runtime["provider"], - runtime["requested_provider"], - runtime["base_url"], - runtime["api_mode"], - runtime["command"], - tuple(runtime["args"]), + model, runtime["provider"], runtime["requested_provider"], runtime["base_url"], + runtime["api_mode"], runtime["command"], tuple(runtime["args"]), ), } - - # Provider-level request_overrides (e.g. a custom_providers extra_body) resolved upstream by - # resolve_runtime_provider(). - service_tier = getattr(self, "_service_tier", None) - if service_tier != "priority": - # None (normal) or auto/cold — the bounded window is applied per - # request by agent.fast_mode, not pinned into request_overrides. + if getattr(self, "_service_tier", None) != "priority": + # None (normal) or auto/cold — the bounded window is applied per request by + # agent.fast_mode, not pinned into request_overrides. route["request_overrides"] = base_request_overrides return route - try: overrides = resolve_fast_mode_overrides( - route["model"], - provider=runtime["provider"], - base_url=runtime["base_url"], + route["model"], provider=runtime["provider"], base_url=runtime["base_url"], ) except Exception: overrides = None - # Fast-mode overrides (service_tier / speed) are top-level keys and do - # not collide with extra_body; deep-merge them over the provider overrides. - route["request_overrides"] = _deep_merge_request_overrides( - base_request_overrides, - overrides or {}, - ) + # Fast-mode keys (service_tier / speed) are top-level and don't collide with extra_body. + route["request_overrides"] = _deep_merge_request_overrides(base_request_overrides, overrides or {}) return route def _sync_session_model_from_agent(self, session_id: str, agent: Any) -> None: - """Persist the runtime model/provider actually used by a gateway turn. - - Provider fallback can switch ``agent.model``/``agent.provider`` after the session row was - created; keep the DB metadata in sync so session lists and tooling report the backend that - actually answered. Runs in the ``run_sync`` closure (executor thread), so it uses the sync - ``SessionDB`` (``_db``) directly rather than the AsyncSessionDB forwarder. - """ + """Persist the runtime model/provider a gateway turn actually used (provider fallback can + switch them after the row was created). Runs in the ``run_sync`` executor thread, so it + uses the sync ``SessionDB`` (``_db``), not the AsyncSessionDB forwarder.""" if not session_id or agent is None or self._session_db is None: return model = getattr(agent, "model", None) @@ -296,13 +227,11 @@ class GatewayTurnMixin: "fallback_active": bool(getattr(agent, "_fallback_activated", False)), } runtime = {k: v for k, v in runtime.items() if v not in (None, "")} - try: db = self._session_db._db row = db.get_session(session_id) if not row: return - current_model = row.get("model") raw_config = row.get("model_config") try: config = json.loads(raw_config) if raw_config else {} @@ -311,9 +240,7 @@ class GatewayTurnMixin: if not isinstance(config, dict): config = {} gateway_runtime = dict(config.get("gateway_runtime") or {}) - if current_model == model and all( - gateway_runtime.get(k) == v for k, v in runtime.items() - ): + if row.get("model") == model and all(gateway_runtime.get(k) == v for k, v in runtime.items()): return config["gateway_runtime"] = runtime db.update_session_meta(session_id, json.dumps(config), model=model) @@ -324,9 +251,8 @@ class GatewayTurnMixin: """Resolve ``source`` to its session entry (topic recovery, internal-route guards, Telegram topic-binding heal). Returns ``(source, session_entry, session_key)`` or ``None`` to drop the event.""" - # Get or create session Topic-mode DMs: rewrite a stale/foreign thread_id to the user's - # last-active topic so a cross-topic Reply or stripped plain reply doesn't fragment the - # conversation across sessions. + # Topic-mode DMs: rewrite a stale/foreign thread_id to the user's last-active topic so a + # cross-topic Reply or stripped plain reply doesn't fragment the conversation. recovered = await asyncio.to_thread(self._recover_telegram_topic_thread_id, source) if recovered is not None: logger.info( @@ -338,26 +264,21 @@ class GatewayTurnMixin: event.source = source event_metadata = getattr(event, "metadata", None) or {} - expected_session_key = str( - event_metadata.get("gateway_session_key") or "" - ).strip() + expected_session_key = str(event_metadata.get("gateway_session_key") or "").strip() if expected_session_key: derived_session_key = self._session_key_for_source(source) if derived_session_key != expected_session_key: logger.warning( "Dropping internally routed event after route recovery: " "expected session=%s derived=%s", - expected_session_key, - derived_session_key, + expected_session_key, derived_session_key, ) return strict_session = bool(event_metadata.get("gateway_session_strict")) pinned_session_id = str(event_metadata.get("gateway_session_id") or "").strip() if strict_session: - session_entry = await self.async_session_store.lookup_by_session_key( - expected_session_key - ) + session_entry = await self.async_session_store.lookup_by_session_key(expected_session_key) if ( session_entry is None or not pinned_session_id @@ -366,109 +287,91 @@ class GatewayTurnMixin: logger.warning( "Dropping internally routed event: expected session id=%s is no " "longer current for key=%s", - pinned_session_id or "missing", - expected_session_key or "missing", + pinned_session_id or "missing", expected_session_key or "missing", ) return else: - # Internal wakes must observe reset policy without becoming user activity themselves. - # Otherwise periodic Kanban/process notifications keep the stable routing key alive - # across every daily/idle boundary. + # Internal wakes must observe reset policy without counting as user activity, or + # periodic Kanban/process notifications keep the routing key alive across every + # daily/idle boundary. session_entry = await self.async_session_store.get_or_create_session( - source, - touch_activity=not bool(getattr(event, "internal", False)), + source, touch_activity=not bool(getattr(event, "internal", False)), ) session_key = session_entry.session_key if not strict_session and pinned_session_id: - resolved_entry = await self._resolve_async_delegation_session( - session_entry, - pinned_session_id, - ) + resolved_entry = await self._resolve_async_delegation_session(session_entry, pinned_session_id) if resolved_entry is None: return session_entry = resolved_entry self._cache_session_source(session_key, source) if await asyncio.to_thread(self._is_telegram_topic_lane, source): - try: - binding = (await self._session_db.get_telegram_topic_binding( - chat_id=str(source.chat_id), - thread_id=str(source.thread_id), - profile_name=self._telegram_topic_profile_name(source), - )) if self._session_db else None - except Exception: - logger.debug("Failed to read Telegram topic binding", exc_info=True) - binding = None - if binding: - bound_session_id = str(binding.get("session_id") or "") - # Heal bindings that point at a pre-compression parent: walk the compression- - # continuation chain forward to its tip so the next message resumes the compressed - # child instead of reloading the oversized parent transcript. - if bound_session_id and self._session_db is not None: - try: - canonical_session_id = await self._session_db.get_compression_tip( - bound_session_id, - ) - except Exception: - logger.debug( - "compression-tip lookup failed for %s", - bound_session_id, exc_info=True, - ) - canonical_session_id = bound_session_id - if ( - canonical_session_id - and canonical_session_id != bound_session_id - ): - bound_session_id = canonical_session_id - if bound_session_id and bound_session_id != session_entry.session_id: - # Route the override through SessionStore so the session_key → session_id - # mapping is persisted to disk and the previous lane session is ended cleanly. - # Mutating session_entry in place created a split-brain: the JSON index pointed - # at one id while downstream code used another. - switched = await self.async_session_store.switch_session(session_key, bound_session_id) - if switched is not None: - session_entry = switched - # If the stored binding pointed at a parent, rewrite it to the - # canonical descendant now that we've followed the chain. - if ( - bound_session_id - and bound_session_id != str(binding.get("session_id") or "") - ): - await asyncio.to_thread( - self._sync_telegram_topic_binding, - source, session_entry, reason="compression-tip-walk", - ) - else: - try: - await asyncio.to_thread(self._record_telegram_topic_binding, source, session_entry) - except Exception: - logger.debug("Failed to record Telegram topic binding", exc_info=True) + session_entry = await self._hmwa_heal_telegram_topic_binding(source, session_entry, session_key) return source, session_entry, session_key + async def _hmwa_heal_telegram_topic_binding(self, source, session_entry, session_key): + """Follow the (chat_id, thread_id) topic binding — healed to its compression tip — or record + a fresh one. Returns the (possibly switched) session entry.""" + try: + binding = (await self._session_db.get_telegram_topic_binding( + chat_id=str(source.chat_id), thread_id=str(source.thread_id), + profile_name=self._telegram_topic_profile_name(source), + )) if self._session_db else None + except Exception: + logger.debug("Failed to read Telegram topic binding", exc_info=True) + binding = None + if not binding: + try: + await asyncio.to_thread(self._record_telegram_topic_binding, source, session_entry) + except Exception: + logger.debug("Failed to record Telegram topic binding", exc_info=True) + return session_entry + stored_session_id = str(binding.get("session_id") or "") + bound_session_id = stored_session_id + # A binding pointing at a pre-compression parent is walked forward to the tip so the next + # message resumes the compressed child instead of reloading the oversized parent. + if bound_session_id and self._session_db is not None: + try: + canonical_session_id = await self._session_db.get_compression_tip(bound_session_id) + except Exception: + logger.debug("compression-tip lookup failed for %s", bound_session_id, exc_info=True) + canonical_session_id = bound_session_id + if canonical_session_id and canonical_session_id != bound_session_id: + bound_session_id = canonical_session_id + if bound_session_id and bound_session_id != session_entry.session_id: + # Route through SessionStore so the session_key → session_id mapping is persisted and + # the previous lane session ended cleanly; mutating session_entry in place split-brained + # the JSON index from downstream code. + switched = await self.async_session_store.switch_session(session_key, bound_session_id) + if switched is not None: + session_entry = switched + if bound_session_id and bound_session_id != stored_session_id: + # The stored binding pointed at a parent: rewrite it to the canonical descendant. + await asyncio.to_thread( + self._sync_telegram_topic_binding, source, session_entry, reason="compression-tip-walk", + ) + return session_entry + async def _hmwa_open_session(self, session_entry, session_key, source): """Consume auto-reset / fresh-reset flags and emit ``session:start`` for new sessions. Returns ``(_was_auto_reset, _is_new_session)``.""" - # Capture and consume was_auto_reset immediately so it cannot re-fire on later messages and - # wipe model/reasoning overrides set between turns. + # Consume was_auto_reset immediately so it cannot re-fire on later messages and wipe + # model/reasoning overrides set between turns. _was_auto_reset = getattr(session_entry, "was_auto_reset", False) if _was_auto_reset: - # Auto-reset is a full conversation boundary: one funnel call clears every conversation- - # scoped per-session dict so the fresh session inherits no model/reasoning overrides, no - # queued "/model switched" note and no stale resolved-model cache. + # Full conversation boundary: one funnel call clears every conversation-scoped + # per-session dict (no inherited overrides, queued "/model switched" note or stale + # resolved-model cache); evict the cached agent (keyed on the stable session_key) so + # context_compressor._previous_summary cannot leak prior history into new summaries. self._clear_conversation_scope(session_key, reason="auto_reset") - # Evict the cached agent: the cache is keyed on the stable session_key, so an auto-reset - # would otherwise reuse the old agent and leak context_compressor._previous_summary - # (prior history) into new compaction summaries. self._evict_cached_agent(session_key) session_entry.was_auto_reset = False - # Emit session:start for new or auto-reset sessions _is_new_session = ( session_entry.created_at == session_entry.updated_at or _was_auto_reset or getattr(session_entry, "is_fresh_reset", False) ) - # Consume the is_fresh_reset flag immediately so it doesn't leak - # onto subsequent messages in the same session (issue #6508). + # Consume is_fresh_reset so it doesn't leak onto later messages in the same session. if getattr(session_entry, "is_fresh_reset", False): session_entry.is_fresh_reset = False if _is_new_session: @@ -485,9 +388,8 @@ class GatewayTurnMixin: from gateway.run import _AUTO_RESET_CONTEXT_NOTES, _auto_reset_reason_text reset_reason = getattr(session_entry, 'auto_reset_reason', None) or 'idle' context_note = _AUTO_RESET_CONTEXT_NOTES.get(reset_reason, _AUTO_RESET_CONTEXT_NOTES["idle"]) - # Slack/Discord channels/threads are long-lived: point the agent at the specific prior - # same-channel session so it recalls that context via session_search instead of an - # unrelated recent session. Deterministic — no extra API/DB calls. + # Long-lived Slack/Discord channels: point the agent at the specific prior same-channel + # session so session_search recalls that context (deterministic, no extra API/DB calls). try: continuity_note = build_channel_continuity_note(session_entry, source) except Exception: @@ -496,51 +398,41 @@ class GatewayTurnMixin: context_note = context_note + "\n\n" + continuity_note turn_sidecar_notes.append(context_note) - # Notify the user about the reset unless notifications are disabled in config, the - # platform is excluded (e.g. api_server, webhook), or the expired session had no - # activity. try: policy = self.session_store.config.get_reset_policy( - platform=source.platform, - session_type=getattr(source, 'chat_type', 'dm'), + platform=source.platform, session_type=getattr(source, 'chat_type', 'dm'), ) platform_name = source.platform.value if source.platform else "" - had_activity = getattr(session_entry, 'reset_had_activity', False) - # Suspended and restart-recovery-expired sessions always notify regardless of - # policy.notify — the user had an active session that was silently replaced, so they - # need to know they can /resume it. Idle/daily resets respect the policy flag. + # Suspended / restart-recovery-expired sessions always notify regardless of + # policy.notify — an active session was silently replaced and the user must learn they + # can /resume it. Idle/daily resets respect the policy flag, the excluded platforms + # (api_server, webhook) and require prior activity. should_notify = reset_reason in {"suspended", "resume_pending_expired"} or ( policy.notify - and had_activity + and getattr(session_entry, 'reset_had_activity', False) and platform_name not in policy.notify_exclude_platforms ) - if should_notify: - adapter = self._adapter_for_source(source) - if adapter: - reason_text = _auto_reset_reason_text(reset_reason, policy) - notice = ( - f"◐ Session automatically reset ({reason_text}). " - f"Conversation history cleared.\n" - f"Use /resume to browse and restore a previous session.\n" - f"Adjust reset timing in config.yaml under session_reset." - ) - try: - session_info = await asyncio.to_thread( - self._reset_notice_session_info, source - ) - if session_info: - notice = f"{notice}\n\n{session_info}" - except Exception: - pass - await adapter.send( - source.chat_id, notice, - metadata=self._thread_metadata_for_source(source), - ) + adapter = self._adapter_for_source(source) if should_notify else None + if adapter: + notice = ( + f"◐ Session automatically reset ({_auto_reset_reason_text(reset_reason, policy)}). " + f"Conversation history cleared.\n" + f"Use /resume to browse and restore a previous session.\n" + f"Adjust reset timing in config.yaml under session_reset." + ) + try: + session_info = await asyncio.to_thread(self._reset_notice_session_info, source) + if session_info: + notice = f"{notice}\n\n{session_info}" + except Exception: + pass + await adapter.send( + source.chat_id, notice, metadata=self._thread_metadata_for_source(source), + ) except Exception as e: logger.debug("Auto-reset notification failed (non-fatal): %s", e) - # was_auto_reset is already consumed in the cleanup block above - # (single source of truth); only the reset reason needs clearing here. + # was_auto_reset was consumed in _hmwa_open_session; only the reason needs clearing. session_entry.auto_reset_reason = None def _hmwa_auto_load_skills(self, event, _auto, _quick_key, session_key): @@ -552,292 +444,199 @@ class GatewayTurnMixin: _loaded_names: list[str] = [] for _sname in _skill_names: _loaded = _load_skill_payload(_sname, task_id=_quick_key) - if _loaded: - _loaded_skill, _skill_dir, _display_name = _loaded - _note = ( - f'[IMPORTANT: The "{_display_name}" skill is auto-loaded. ' - f"Follow its instructions for this session.]" - ) - _part = _build_skill_message(_loaded_skill, _skill_dir, _note) - if _part: - _combined_parts.append(_part) - _loaded_names.append(_sname) - else: + if not _loaded: logger.warning("[Gateway] Auto-skill '%s' not found", _sname) - if _combined_parts: - # Append the user's original text after all skill payloads - _combined_parts.append(event.text) - event.text = "\n\n".join(_combined_parts) - logger.info( - "[Gateway] Auto-loaded skill(s) %s for session %s", - _loaded_names, session_key, + continue + _loaded_skill, _skill_dir, _display_name = _loaded + _part = _build_skill_message( + _loaded_skill, _skill_dir, + f'[IMPORTANT: The "{_display_name}" skill is auto-loaded. ' + f"Follow its instructions for this session.]", ) + if _part: + _combined_parts.append(_part) + _loaded_names.append(_sname) + if _combined_parts: + _combined_parts.append(event.text) # user's original text after the payloads + event.text = "\n\n".join(_combined_parts) + logger.info("[Gateway] Auto-loaded skill(s) %s for session %s", _loaded_names, session_key) except Exception as e: logger.warning("[Gateway] Failed to auto-load skill(s) %s: %s", _skill_names, e) async def _hmwa_acquire_turn_lease(self, _quick_key, run_generation, session_entry, _session_env_tokens): - # ── Turn lease: session resolution is FINAL here. Serialize [load history → run → flush] - # per resolved SESSION_ID: another routing key mapped to the same session_id waits for the - # prior flush instead of loading a stale base and interleaving writes (same-key messages - # never reach here mid-turn thanks to adapter + runner guards, so the lock is otherwise - # uncontended). Fail-closed on timeout: never enter the transcript region without a lease; - # outer dispatch returns a bounded resend notice. Released in _handle_message's finally - # (_release_turn_lease), granted per (routing key, run generation) so a stale unwind can't - # release a newer turn's. + """Serialize [load history → run → flush] per resolved SESSION_ID (session resolution is + FINAL here): another routing key mapped to the same session_id waits for the prior flush + instead of loading a stale base. Fail-closed on timeout — never enter the transcript + region without a lease; outer dispatch returns a bounded resend notice. Released in + _handle_message's finally, granted per (routing key, run generation) so a stale unwind + can't release a newer turn's.""" from gateway.run import _float_env _lease_registry = getattr(self, "_turn_leases", None) - if _lease_registry is not None: - try: - _lease_token = await _lease_registry.acquire( - session_entry.session_id, - owner_key=_quick_key, - generation=run_generation, - timeout=_float_env( - "HERMES_TURN_LEASE_TIMEOUT", DEFAULT_LEASE_WAIT - ), - ) - except TurnLeaseTimeoutError: - # The broad session-context cleanup finally starts later in this - # method. Restore the tokens here before propagating the rejection - # to outer dispatch, or this early exit leaks task-local identity. - self._clear_session_env(_session_env_tokens) - raise - if _lease_token is not None: - _lease_state = self._session_state(_quick_key).turn - _lease_state.lease_token = _lease_token - _lease_state.lease_generation = run_generation + if _lease_registry is None: + return + try: + _lease_token = await _lease_registry.acquire( + session_entry.session_id, owner_key=_quick_key, generation=run_generation, + timeout=_float_env("HERMES_TURN_LEASE_TIMEOUT", DEFAULT_LEASE_WAIT), + ) + except TurnLeaseTimeoutError: + # The broad session-context cleanup finally starts later; restore the tokens here or + # this early exit leaks task-local identity. + self._clear_session_env(_session_env_tokens) + raise + if _lease_token is not None: + _lease_state = self._session_state(_quick_key).turn + _lease_state.lease_token = _lease_token + _lease_state.lease_generation = run_generation + + @dataclasses.dataclass + class _HygienePlan: + """Hygiene pre-check outcome for one turn.""" + + needs_compress: bool + approx_tokens: int + msg_count: int + warn_token_threshold: int async def _hmwa_hygiene_settings(self, source, session_key): """Resolve model/provider/context-length + hygiene knobs for the pre-agent compression - safety net (fail-soft: any config/runtime error keeps the defaults).""" + safety net (fail-soft: any config/runtime error keeps the defaults). + + The hygiene threshold (0.85) is deliberately HIGHER than the agent's own compressor + (0.50): it is a safety net for sessions that grew between turns. ``max_turn_hold_seconds`` + bounds how long the user's TURN waits on hygiene before proceeding uncompressed (the + compressor keeps running detached, commit fenced); kept below transport idle-timeouts.""" from gateway.run import _load_gateway_config - # Read model + compression config. Hygiene threshold is intentionally HIGHER than the - # agent's own compressor (0.85 vs 0.50): it is a pre-agent safety net for sessions that - # grew between turns; at 0.50 it compressed prematurely on every turn in long sessions. - _hyg_model = "anthropic/claude-sonnet-4.6" - _hyg_threshold_pct = 0.85 - _hyg_compression_enabled = True - _hyg_hard_msg_limit = 5000 - _hyg_timeout_seconds = 30.0 - _hyg_total_ceiling_seconds = 600.0 - # Max wall-clock the user's TURN is held waiting on hygiene compression before the - # gateway stops waiting and proceeds on the uncompressed transcript. The compressor keeps - # running detached; its commit is fenced (revoke_commit_admission) so a stale result can - # never clobber later turns. Kept well below transport idle-timeouts (Telegram ~30s). - _hyg_max_turn_hold_seconds = 10.0 - _hyg_failure_cooldown_seconds = 300.0 - _hyg_config_context_length = None - _hyg_provider = None - _hyg_base_url = None - _hyg_api_key = None - _hyg_configured_model = None - _hyg_configured_provider = None - _hyg_configured_base_url = None - _hyg_data = {} + hs = self._HygieneSettings( + model="anthropic/claude-sonnet-4.6", threshold_pct=0.85, compression_enabled=True, + hard_msg_limit=5000, timeout_seconds=30.0, total_ceiling_seconds=600.0, + max_turn_hold_seconds=10.0, failure_cooldown_seconds=300.0, config_context_length=None, + provider=None, base_url=None, api_key=None, data={}, + ) try: - _hyg_data = _load_gateway_config() - if _hyg_data: + hs.data = _load_gateway_config() + if hs.data: # Resolve model name (same logic as run_sync) - _model_cfg = _hyg_data.get("model", {}) + _model_cfg = hs.data.get("model", {}) if isinstance(_model_cfg, str): - _hyg_model = _model_cfg + hs.model = _model_cfg elif isinstance(_model_cfg, dict): - _hyg_model = _model_cfg.get("default") or _model_cfg.get("model") or _hyg_model - # Read explicit context_length override from model config - # (same as run_agent.py lines 995-1005) + hs.model = _model_cfg.get("default") or _model_cfg.get("model") or hs.model _raw_ctx = _model_cfg.get("context_length") if _raw_ctx is not None: with suppress(TypeError, ValueError): - _hyg_config_context_length = int(_raw_ctx) - # Read provider for accurate context detection - _hyg_provider = _model_cfg.get("provider") or None - _hyg_base_url = _model_cfg.get("base_url") or None + hs.config_context_length = int(_raw_ctx) + hs.provider = _model_cfg.get("provider") or None + hs.base_url = _model_cfg.get("base_url") or None - # Only the enabled flag is read; hygiene's threshold is deliberately separate - # from the agent's compression.threshold (hygiene runs higher). - _comp_cfg = _hyg_data.get("compression", {}) + # Only the enabled flag is shared with the agent's compression config; hygiene's + # threshold is deliberately separate (runs higher). + _comp_cfg = hs.data.get("compression", {}) if isinstance(_comp_cfg, dict): - _hyg_compression_enabled = str( - _comp_cfg.get("enabled", True) - ).lower() in {"true", "1", "yes"} - _raw_hard_limit = _comp_cfg.get("hygiene_hard_message_limit") - if _raw_hard_limit is not None: - try: - _parsed = int(_raw_hard_limit) - if _parsed > 0: - _hyg_hard_msg_limit = _parsed - except (TypeError, ValueError): - pass - _raw_timeout = _comp_cfg.get("hygiene_timeout_seconds") - if _raw_timeout is not None: - try: - _parsed = float(_raw_timeout) - if _parsed > 0: - _hyg_timeout_seconds = _parsed - except (TypeError, ValueError): - pass - _raw_ceiling = _comp_cfg.get("hygiene_total_ceiling_seconds") - if _raw_ceiling is not None: - try: - _parsed = float(_raw_ceiling) - if _parsed > 0: - _hyg_total_ceiling_seconds = _parsed - except (TypeError, ValueError): - pass - # The ceiling can never be tighter than one idle - # window, or the extension loop would be dead code. - _hyg_total_ceiling_seconds = max( - _hyg_total_ceiling_seconds, _hyg_timeout_seconds, - ) - _raw_turn_hold = _comp_cfg.get("hygiene_max_turn_hold_seconds") - if _raw_turn_hold is not None: - try: - _parsed = float(_raw_turn_hold) - if _parsed > 0: - _hyg_max_turn_hold_seconds = _parsed - except (TypeError, ValueError): - pass - _raw_cooldown = _comp_cfg.get("hygiene_failure_cooldown_seconds") - if _raw_cooldown is not None: - try: - _parsed = float(_raw_cooldown) - if _parsed >= 0: - _hyg_failure_cooldown_seconds = _parsed - except (TypeError, ValueError): - pass + hs.compression_enabled = str(_comp_cfg.get("enabled", True)).lower() in {"true", "1", "yes"} - _hyg_configured_model = _hyg_model - _hyg_configured_provider = _hyg_provider - _hyg_configured_base_url = _hyg_base_url + def _knob(key, current, cast, allow_zero=False): + raw = _comp_cfg.get(key) + if raw is None: + return current + try: + parsed = cast(raw) + except (TypeError, ValueError): + return current + return parsed if (parsed >= 0 if allow_zero else parsed > 0) else current + + hs.hard_msg_limit = _knob("hygiene_hard_message_limit", hs.hard_msg_limit, int) + hs.timeout_seconds = _knob("hygiene_timeout_seconds", hs.timeout_seconds, float) + hs.total_ceiling_seconds = _knob("hygiene_total_ceiling_seconds", hs.total_ceiling_seconds, float) + # The ceiling can never be tighter than one idle window, or the extension + # loop would be dead code. + hs.total_ceiling_seconds = max(hs.total_ceiling_seconds, hs.timeout_seconds) + hs.max_turn_hold_seconds = _knob("hygiene_max_turn_hold_seconds", hs.max_turn_hold_seconds, float) + hs.failure_cooldown_seconds = _knob( + "hygiene_failure_cooldown_seconds", hs.failure_cooldown_seconds, float, allow_zero=True, + ) + + configured_model, configured_provider, configured_base_url = hs.model, hs.provider, hs.base_url try: - _hyg_model, _hyg_runtime = self._resolve_session_agent_runtime( - source=source, - session_key=session_key, - user_config=_hyg_data if isinstance(_hyg_data, dict) else None, + hs.model, _hyg_runtime = self._resolve_session_agent_runtime( + source=source, session_key=session_key, + user_config=hs.data if isinstance(hs.data, dict) else None, ) - _hyg_provider = _hyg_runtime.get("provider") or _hyg_provider - _hyg_base_url = _hyg_runtime.get("base_url") or _hyg_base_url - _hyg_api_key = _hyg_runtime.get("api_key") or _hyg_api_key + hs.provider = _hyg_runtime.get("provider") or hs.provider + hs.base_url = _hyg_runtime.get("base_url") or hs.base_url + hs.api_key = _hyg_runtime.get("api_key") or hs.api_key except Exception: pass - if _hyg_config_context_length is not None: + if hs.config_context_length is not None: try: from hermes_cli.route_identity import should_clear_context_pin_async if await should_clear_context_pin_async( - _hyg_configured_model, - _hyg_model, - _hyg_configured_base_url, - _hyg_base_url, - _hyg_configured_provider, - _hyg_provider, + configured_model, hs.model, configured_base_url, hs.base_url, + configured_provider, hs.provider, ): - _hyg_config_context_length = None + hs.config_context_length = None except Exception: - _hyg_config_context_length = None + hs.config_context_length = None # custom_providers per-model context_length fallback (as in run_agent.py); must run - # after runtime resolution so _hyg_base_url is set. - if _hyg_config_context_length is None and _hyg_base_url: + # after runtime resolution so base_url is set. + if hs.config_context_length is None and hs.base_url: try: try: from hermes_cli.config import ( get_compatible_custom_providers as _gw_gcp, get_custom_provider_context_length as _gw_gccl, ) - _hyg_custom_providers = _gw_gcp(_hyg_data) + _hyg_custom_providers = _gw_gcp(hs.data) except Exception: - _hyg_custom_providers = _hyg_data.get("custom_providers") + _hyg_custom_providers = hs.data.get("custom_providers") if not isinstance(_hyg_custom_providers, list): _hyg_custom_providers = [] _hyg_custom_ctx = _gw_gccl( - model=_hyg_model, - base_url=_hyg_base_url, - custom_providers=_hyg_custom_providers, + model=hs.model, base_url=hs.base_url, custom_providers=_hyg_custom_providers, ) if _hyg_custom_ctx: - _hyg_config_context_length = int(_hyg_custom_ctx) + hs.config_context_length = int(_hyg_custom_ctx) except (TypeError, ValueError): pass except Exception: pass - return self._HygieneSettings( - model=_hyg_model, - threshold_pct=_hyg_threshold_pct, - compression_enabled=_hyg_compression_enabled, - hard_msg_limit=_hyg_hard_msg_limit, - timeout_seconds=_hyg_timeout_seconds, - total_ceiling_seconds=_hyg_total_ceiling_seconds, - max_turn_hold_seconds=_hyg_max_turn_hold_seconds, - failure_cooldown_seconds=_hyg_failure_cooldown_seconds, - config_context_length=_hyg_config_context_length, - provider=_hyg_provider, - base_url=_hyg_base_url, - api_key=_hyg_api_key, - data=_hyg_data, - ) + return hs async def _hmwa_hygiene_plan(self, hs, history, session_entry, session_key): """Decide whether hygiene compression fires this turn (token/message thresholds, DB-backed - failure cooldown, in-flight compression). Returns - ``(_needs_compress, _approx_tokens, _msg_count, _warn_token_threshold)``.""" - from agent.model_metadata import ( - estimate_messages_tokens_rough, - get_model_context_length_async, - ) - - _hyg_model = hs.model - _hyg_threshold_pct = hs.threshold_pct - _hyg_hard_msg_limit = hs.hard_msg_limit - _hyg_config_context_length = hs.config_context_length - _hyg_provider = hs.provider - _hyg_base_url = hs.base_url - _hyg_api_key = hs.api_key + failure cooldown, in-flight compression).""" + from agent.model_metadata import estimate_messages_tokens_rough, get_model_context_length_async _hyg_context_length = await get_model_context_length_async( - _hyg_model, - base_url=_hyg_base_url or "", - api_key=_hyg_api_key or "", - config_context_length=_hyg_config_context_length, - provider=_hyg_provider or "", - ) - _compress_token_threshold = int( - _hyg_context_length * _hyg_threshold_pct + hs.model, base_url=hs.base_url or "", api_key=hs.api_key or "", + config_context_length=hs.config_context_length, provider=hs.provider or "", ) + _compress_token_threshold = int(_hyg_context_length * hs.threshold_pct) _warn_token_threshold = int(_hyg_context_length * 0.95) - _msg_count = len(history) - # Prefer actual API-reported tokens from the last turn - # (stored in session entry) over the rough char-based estimate. - _stored_tokens = session_entry.last_prompt_tokens - if _stored_tokens > 0: - _approx_tokens = _stored_tokens - _token_source = "actual" + # Prefer the API-reported prompt tokens from the last turn over the rough char estimate. + # Rough estimates run 30-50% high on code/JSON-heavy sessions, which only makes hygiene + # fire early (safe). Do NOT compensate with a threshold multiplier: 85% * 1.4 = 119% of + # context kept hygiene from ever firing for ~200K models. + if session_entry.last_prompt_tokens > 0: + _approx_tokens, _token_source = session_entry.last_prompt_tokens, "actual" else: - _approx_tokens = estimate_messages_tokens_rough(history) - _token_source = "estimated" - # Rough estimates run 30-50% high on code/JSON-heavy sessions, which only makes - # hygiene fire early (safe). Do NOT compensate with a threshold multiplier: 85% - # * 1.4 = 119% of context kept hygiene from ever firing for ~200K models. + _approx_tokens, _token_source = estimate_messages_tokens_rough(history), "estimated" # Hard safety valve: force compression at an extreme message count regardless of token - # estimates, breaking the spiral where API disconnects prevent token data → no - # compression → more disconnects. Default 5000 sits clear of legitimate 1M+ context - # sessions (those compress on tokens). Config: compression.hygiene_hard_message_limit. - _HARD_MSG_LIMIT = _hyg_hard_msg_limit - _needs_compress = ( - _approx_tokens >= _compress_token_threshold - or _msg_count >= _HARD_MSG_LIMIT - ) + # estimates, breaking the spiral where API disconnects prevent token data → no compression + # → more disconnects. Default 5000 sits clear of legitimate 1M+ context sessions. + _needs_compress = _approx_tokens >= _compress_token_threshold or _msg_count >= hs.hard_msg_limit if _needs_compress: - # Use the persistent DB-backed cooldown (same as the in-conversation compression - # path in context_compressor.py) so the cooldown survives gateway restarts. The - # in-memory dict reset on every restart, re-triggering the same failing - # compression and wedging session storage. + # Persistent DB-backed cooldown (shared with context_compressor.py) so it survives + # gateway restarts; an in-memory dict re-triggered the same failing compression on + # every restart and wedged session storage. _session_db = getattr(self, "_session_db", None) if _session_db is not None: _session_db = getattr(_session_db, "_db", _session_db) @@ -851,22 +650,17 @@ class GatewayTurnMixin: logger.info( "Session hygiene: skipping compression for %s; " "previous failure cooldown active for %.1fs", - session_entry.session_id, - _cooldown_state["remaining_seconds"], + session_entry.session_id, _cooldown_state["remaining_seconds"], ) _needs_compress = False - if _needs_compress and await self._session_has_compression_in_flight( - session_key - ): - # A prior hygiene/agent compression still holds the durable lock (typically a - # shielded worker left behind by /stop or /restart). Starting another attempt - # would wait up to the 600s ceiling behind a commit the fence will refuse, while - # inbound messages demote to queue. + if _needs_compress and await self._session_has_compression_in_flight(session_key): + # A prior hygiene/agent compression still holds the durable lock (typically a shielded + # worker left behind by /stop or /restart). Another attempt would wait up to the 600s + # ceiling behind a commit the fence will refuse, while inbound messages demote to queue. logger.info( "Session hygiene: skipping compression for %s; " - "another compression is already in flight", - session_entry.session_id, + "another compression is already in flight", session_entry.session_id, ) _needs_compress = False @@ -875,628 +669,324 @@ class GatewayTurnMixin: "Session hygiene: %s messages, ~%s tokens (%s) — auto-compressing " "(threshold: %s%% of %s = %s tokens)", _msg_count, f"{_approx_tokens:,}", _token_source, - int(_hyg_threshold_pct * 100), - f"{_hyg_context_length:,}", - f"{_compress_token_threshold:,}", + int(hs.threshold_pct * 100), f"{_hyg_context_length:,}", f"{_compress_token_threshold:,}", ) - return _needs_compress, _approx_tokens, _msg_count, _warn_token_threshold + return self._HygienePlan(_needs_compress, _approx_tokens, _msg_count, _warn_token_threshold) async def _hmwa_hygiene_wait_for_summary(self, attempt, hs, session_entry): """Progress-aware inline wait for the detached hygiene compressor. Returns the compressed transcript; raises ``HygieneTurnHoldExceeded`` (turn-hold budget) or - ``asyncio.TimeoutError`` (idle/ceiling/fence cancel) for the caller's handlers.""" - from gateway.run import HygieneTurnHoldExceeded, hygiene_wait_should_extend - _hyg_commit_fence = attempt.commit_fence - _hyg_future = attempt.future - _hyg_wait_started = attempt.wait_started - _hyg_timeout_seconds = hs.timeout_seconds - _hyg_total_ceiling_seconds = hs.total_ceiling_seconds - _hyg_max_turn_hold_seconds = hs.max_turn_hold_seconds + ``asyncio.TimeoutError`` (idle/ceiling/fence cancel) for the caller's handlers. - # Progress-aware wait: the timeout is an INACTIVITY budget — - # the worker ticks the fence per streamed token, so a slow but - # still-generating model extends the deadline. A hard ceiling - # bounds the total so a trickle stream can't hold the turn. + The timeout is an INACTIVITY budget — the worker ticks the fence per streamed token, so a + slow but still-generating model extends the deadline; a hard ceiling bounds the total so + a trickle stream can't hold the turn; the turn-hold budget caps how long the user's TURN + waits regardless.""" + from gateway.run import HygieneTurnHoldExceeded, hygiene_wait_should_extend + fence = attempt.commit_fence while True: - if _hyg_commit_fence.is_cancelled: + if fence.is_cancelled: raise asyncio.TimeoutError - # Charge the idle budget from the LAST PROGRESS event, - # not from the start of this wait slice — otherwise - # silence can approach 2x the configured timeout. - _hyg_waited = ( - time.monotonic() - _hyg_wait_started - ) + # Charge the idle budget from the LAST PROGRESS event, not from the start of this wait + # slice — otherwise silence can approach 2x the configured timeout. + _hyg_waited = time.monotonic() - attempt.wait_started _slice = min( - max( - _hyg_timeout_seconds - - _hyg_commit_fence.seconds_since_progress(), - 0.005, - ), - max( - _hyg_total_ceiling_seconds - - _hyg_waited, - 0.005, - ), - ) - # Bounded turn-hold: cap this slice at the remaining - # turn-hold budget so it is re-checked against - # _hyg_max_turn_hold_seconds at least that often — - # otherwise a continuously-streaming worker keeps the - # slice large and holds the turn until the ceiling. - _turn_hold_remaining = ( - _hyg_max_turn_hold_seconds - - (time.monotonic() - _hyg_wait_started) - ) - if _turn_hold_remaining <= 0: - # Budget exhausted: force an immediate timeout so - # the abandonment path below runs. - _slice = 0.005 - else: - _slice = min( - _slice, - max(_turn_hold_remaining, 0.005), - ) - # Re-check the fence on a short poll so a - # /stop or /restart cancel is not stuck - # behind a full idle window (#96953). - _idle_left = max( - _hyg_timeout_seconds - - _hyg_commit_fence.seconds_since_progress(), - 0.005, + max(hs.timeout_seconds - fence.seconds_since_progress(), 0.005), + max(hs.total_ceiling_seconds - _hyg_waited, 0.005), ) + # Cap the slice at the remaining turn-hold budget so it is re-checked at least that + # often — a continuously-streaming worker would otherwise keep the slice large and hold + # the turn until the ceiling. Budget exhausted → immediate timeout → abandonment path. + _turn_hold_remaining = hs.max_turn_hold_seconds - (time.monotonic() - attempt.wait_started) + _slice = 0.005 if _turn_hold_remaining <= 0 else min(_slice, max(_turn_hold_remaining, 0.005)) + # Short poll so a /stop or /restart cancel is not stuck behind a full idle window. + _idle_left = max(hs.timeout_seconds - fence.seconds_since_progress(), 0.005) _slice = min(_slice, 0.25) try: - _compressed, _ = await asyncio.wait_for( - asyncio.shield(_hyg_future), - timeout=_slice, - ) - break + _compressed, _ = await asyncio.wait_for(asyncio.shield(attempt.future), timeout=_slice) + return _compressed except asyncio.TimeoutError: - if _hyg_commit_fence.is_cancelled: + if fence.is_cancelled: raise - _hyg_waited = time.monotonic() - _hyg_wait_started - _idle = _hyg_commit_fence.seconds_since_progress() - # Bounded turn-hold: never hold the user's TURN past - # _hyg_max_turn_hold_seconds even if the summary - # model is still streaming; fall through to the - # timeout path, which revokes commit admission and - # proceeds on the uncompressed transcript, so the - # wire never trips a transport idle-timeout. - if ( - _hyg_waited - >= _hyg_max_turn_hold_seconds - ): + _hyg_waited = time.monotonic() - attempt.wait_started + _idle = fence.seconds_since_progress() + # Never hold the user's TURN past the budget even if the summary model is still + # streaming: the turn-hold path proceeds on the uncompressed transcript so the wire + # never trips a transport idle-timeout. + if _hyg_waited >= hs.max_turn_hold_seconds: logger.info( - "Session hygiene compression for " - "session %s exceeded the turn-hold " - "budget (%.1fs >= %.1fs) — " - "abandoning inline wait, proceeding " + "Session hygiene compression for session %s exceeded the turn-hold " + "budget (%.1fs >= %.1fs) — abandoning inline wait, proceeding " "without compression this turn", - session_entry.session_id, - _hyg_waited, - _hyg_max_turn_hold_seconds, + session_entry.session_id, _hyg_waited, hs.max_turn_hold_seconds, ) raise HygieneTurnHoldExceeded( - f"turn-hold budget {_hyg_max_turn_hold_seconds:.1f}s " + f"turn-hold budget {hs.max_turn_hold_seconds:.1f}s " f"elapsed after {_hyg_waited:.1f}s" ) if hygiene_wait_should_extend( - idle=_idle, - timeout=_hyg_timeout_seconds, - waited=_hyg_waited, - ceiling=_hyg_total_ceiling_seconds, - fence_cancelled=_hyg_commit_fence.is_cancelled, + idle=_idle, timeout=hs.timeout_seconds, waited=_hyg_waited, + ceiling=hs.total_ceiling_seconds, fence_cancelled=fence.is_cancelled, ): if _slice >= _idle_left - 1e-9: logger.info( - "Session hygiene compression for " - "session %s still streaming after " - "%.0fs (last progress %.1fs ago) — " - "extending wait (ceiling %.0fs)", - session_entry.session_id, - _hyg_waited, _idle, - _hyg_total_ceiling_seconds, + "Session hygiene compression for session %s still streaming after " + "%.0fs (last progress %.1fs ago) — extending wait (ceiling %.0fs)", + session_entry.session_id, _hyg_waited, _idle, hs.total_ceiling_seconds, ) continue raise - return _compressed + + async def _hmwa_hygiene_cancel_before_commit(self, fence): + """Cancel the detached worker at the commit fence. Returns ``False`` when the worker had + already crossed into its commit (caller must consume the result — aborting mid-commit + would corrupt the message-store transaction), ``True`` when cancelled in time. + + A hung commit retains the fence lock; the lock-free ``commit_in_flight`` marker keeps + this loop from spinning forever. 25ms polls ride transient lock-setup windows.""" + while True: + if fence.commit_in_flight: + return False + cancelled = fence.try_cancel_before_commit() + if cancelled is not None: + return cancelled + await asyncio.sleep(0.025) + + def _hmwa_hygiene_defer_cleanup(self, attempt, context): + """Hand the agent's cleanup to the still-running worker future and mark it deferred.""" + self._defer_agent_cleanup_until_future_done(attempt.future, attempt.agent, context=context) + attempt.cleanup_deferred = True + + @staticmethod + def _hmwa_hygiene_stamp(agent, desc, provenance_name, debug_label): + from agent.session_activity import ActivityProvenance + from gateway.run import _stamp_hygiene_compression_provenance + _stamp_hygiene_compression_provenance( + agent, desc, getattr(ActivityProvenance, provenance_name), debug_label, + ) + + async def _hmwa_hygiene_notify(self, source, meta, message, what): + """Best-effort user notice on the hygiene thread; failure is logged, never raised.""" + try: + _adapter = self._adapter_for_source(source) + if _adapter and source.chat_id: + await _adapter.send(source.chat_id, message, metadata=meta) + except Exception as _werr: + logger.warning("Failed to deliver %s to user: %s", what, _werr) + + async def _hmwa_hygiene_record_failure_cooldown(self, hs, session_key, session_id, reason): + """Escalate the failure streak (off-loop) and persist the cooldown, when enabled.""" + from gateway.run import _hygiene_cooldown_for_failure, _record_hygiene_cooldown + if hs.failure_cooldown_seconds < 0: + return + _hyg_cooldown = await asyncio.to_thread( + _hygiene_cooldown_for_failure, self, session_key, hs.failure_cooldown_seconds, + ) + _record_hygiene_cooldown(self, session_id, _hyg_cooldown, reason) async def _hmwa_hygiene_on_turn_hold(self, attempt, hs, session_entry, session_key, source): """``except HygieneTurnHoldExceeded`` body: keep or cancel the worker's commit admission, notify the user, and re-raise; returns the compressed transcript only when the worker - was already committing.""" + was already committing. + + Turn-hold expiry is an availability boundary, not a failure: the compressor is healthy, + so the failure streak must NOT advance — only flat, non-escalating retry spacing is + recorded. When the worker's commit is watermark-fenced (rows appended after compression + start — this turn included — survive its late commit as cloned concurrent tail) the + attempt KEEPS commit admission: the turn proceeds uncompressed NOW and the summary is + adopted at the worker's own fenced commit; always cancelling burned every attempt for + thinking summary models whose reasoning prefix alone exceeds the hold. Without the + watermark fence (no session_db, capture failed, legacy lock API) a late commit could + clobber newer turns, so cancel.""" from gateway.run import ( - _HYGIENE_TURNHOLD_RETRY_SECONDS, - _record_hygiene_cooldown, - _reset_hygiene_failure_streak, - _stamp_hygiene_compression_provenance, + _HYGIENE_TURNHOLD_RETRY_SECONDS, _record_hygiene_cooldown, _reset_hygiene_failure_streak + ) + fence = attempt.commit_fence + _hyg_keep_admission = ( + bool(getattr(fence, "commit_watermark_fenced", False)) and not fence.is_cancelled ) - _hyg_agent = attempt.agent - _hyg_meta = attempt.meta - _hyg_commit_fence = attempt.commit_fence - _hyg_future = attempt.future - _hyg_wait_started = attempt.wait_started - _hyg_max_turn_hold_seconds = hs.max_turn_hold_seconds - - # Turn-hold expiry is an availability boundary, not a - # failure: the compressor is healthy and still streaming; we - # just can't hold the turn longer. Share the safe mechanics - # (fence, release, defer, proceed uncompressed) with - # distinct provenance / user message and NO failure-cooldown - # increment. Decouple the TURN from the COMPRESSION: when - # the worker's commit is watermark-fenced (rows appended - # after compression start, this turn included, survive its - # late commit as cloned concurrent tail) the attempt KEEPS - # commit admission — the turn proceeds uncompressed NOW and - # the summary is adopted at the worker's own fenced commit - # (archive_and_compact / rotation publish); always - # cancelling burned every attempt for thinking summary - # models whose reasoning prefix alone exceeds the hold. If - # NOT watermark-fenced (no session_db, capture failed, - # legacy lock API) a late commit could clobber newer turns, - # so cancel. - _hyg_keep_admission = bool( - getattr( - _hyg_commit_fence, - "commit_watermark_fenced", - False, - ) - ) and not _hyg_commit_fence.is_cancelled if _hyg_keep_admission: - self._defer_agent_cleanup_until_future_done( - _hyg_future, - _hyg_agent, - context="session hygiene turn-hold", - ) - attempt.cleanup_deferred = True - # NO retry-after here: the attempt is still running - # toward a real commit, and the flat 60s retry-after - # would also block the agent-side preflight compressor - # (same-session cooldown). Re-attempt spacing comes from - # the durable compression lock instead: the next turn's - # hygiene pre-check skips while this worker's lease is - # held (_session_has_compression_in_flight). The - # done-callback below records the flat retry-after ONLY - # if the worker ends without committing anything. - _hyg_deferred_sid = session_entry.session_id - _hyg_deferred_key = session_key - _hyg_deferred_agent = _hyg_agent + self._hmwa_hygiene_defer_cleanup(attempt, "session hygiene turn-hold") + # NO retry-after here: the attempt is still running toward a real commit, and the flat + # 60s retry-after would also block the agent-side preflight compressor (same-session + # cooldown). Re-attempt spacing comes from the durable compression lock: the next + # turn's hygiene pre-check skips while this worker's lease is held. The done-callback + # records the flat retry-after ONLY if the worker ends without committing anything. + _sid, _skey, _agent = session_entry.session_id, session_key, attempt.agent - def _hyg_adopt_or_space_retry( - _fut, - _gw=self, - _sid=_hyg_deferred_sid, - _skey=_hyg_deferred_key, - _agent=_hyg_deferred_agent, - ): + def _hyg_adopt_or_space_retry(_fut, _gw=self, _sid=_sid, _skey=_skey, _agent=_agent): try: _exc = _fut.exception() - except ( - asyncio.CancelledError, - Exception, - ): - _exc = None + except (asyncio.CancelledError, Exception): _committed = False else: _committed = _exc is None and ( - bool( - getattr( - _agent, - "_last_compaction_in_place", - False, - ) - ) - or getattr( - _agent, "session_id", _sid - ) - != _sid + bool(getattr(_agent, "_last_compaction_in_place", False)) + or getattr(_agent, "session_id", _sid) != _sid ) if _committed: logger.info( - "Session hygiene compression for " - "session %s finished after the " - "turn-hold was released — summary " - "adopted at the watermark-fenced " - "commit boundary (#97963)", - _sid, + "Session hygiene compression for session %s finished after the " + "turn-hold was released — summary adopted at the watermark-fenced " + "commit boundary (#97963)", _sid, ) try: - _reset_hygiene_failure_streak( - _gw, _skey - ) + _reset_hygiene_failure_streak(_gw, _skey) except Exception as _rs_err: - logger.debug( - "hygiene streak reset after " - "deferred adoption failed: %s", - _rs_err, - ) + logger.debug("hygiene streak reset after deferred adoption failed: %s", _rs_err) else: - # Nothing to adopt (summary failed, fence - # refused the commit, or the attempt was - # superseded). Record flat spacing so sustained - # traffic does not spawn and abandon a fresh - # compressor every turn; non-escalating — the - # failure streak must not advance for a deferral. + # Nothing to adopt (summary failed, fence refused the commit, or the attempt + # was superseded). Flat spacing so sustained traffic does not spawn and + # abandon a fresh compressor every turn. _record_hygiene_cooldown( - _gw, _sid, - _HYGIENE_TURNHOLD_RETRY_SECONDS, - "hygiene compression deferred: " - "turn-hold budget expired and the " + _gw, _sid, _HYGIENE_TURNHOLD_RETRY_SECONDS, + "hygiene compression deferred: turn-hold budget expired and the " "detached attempt did not commit", ) - _hyg_future.add_done_callback( - _hyg_adopt_or_space_retry + attempt.future.add_done_callback(_hyg_adopt_or_space_retry) + _log_suffix = ( + " — the watermark-fenced worker keeps its commit admission and the summary " + "will be adopted when it finishes" ) - from agent.session_activity import ( - ActivityProvenance, - ) - _stamp_hygiene_compression_provenance( - _hyg_agent, - "session hygiene compression turn-hold", - ActivityProvenance.AGENT_COMPRESSION_TURNHOLD, - "hygiene compression turn-hold " - "activity stamp failed", - ) - logger.info( - "Session hygiene compression for session %s " - "exceeded turn-hold budget (%.1fs); " - "proceeding without compression this turn — " - "the watermark-fenced worker keeps its " - "commit admission and the summary will be " - "adopted when it finishes", - session_entry.session_id, - time.monotonic() - _hyg_wait_started, - ) - _turnhold_msg = t( - "gateway.compress.turnhold_deferred" - ) - try: - _adapter = self._adapter_for_source(source) - if _adapter and source.chat_id: - await _adapter.send( - source.chat_id, - _turnhold_msg, - metadata=_hyg_meta, - ) - except Exception as _werr: - logger.warning( - "Failed to deliver compression-turnhold " - "notice to user: %s", - _werr, - ) - raise - _cancelled = None - while _cancelled is None: - if _hyg_commit_fence.commit_in_flight: - _cancelled = False - break - _cancelled = ( - _hyg_commit_fence.try_cancel_before_commit() - ) - if _cancelled is None: - await asyncio.sleep(0.025) - if not _cancelled: - # NOTE: bounded overshoot by design: the turn can be held - # past _hyg_max_turn_hold_seconds by up to the commit - # duration. Aborting mid-commit would corrupt the - # message-store transaction — the overshoot is the - # cheaper failure mode. Do NOT "fix" this into a - # mid-commit cancellation. - _compressed, _ = await _hyg_future else: - _hyg_commit_fence.release_cancelled_compression_lock() - self._defer_agent_cleanup_until_future_done( - _hyg_future, - _hyg_agent, - context="session hygiene turn-hold", - ) - attempt.cleanup_deferred = True - # Short, NON-escalating retry-after. Without it every - # turn re-spawns a compressor, holds it for the turn-hold - # budget and cancels it — token burn that never commits. - # Deliberately NOT _hygiene_cooldown_for_failure: the - # compressor is healthy, so the failure streak must not - # advance; only flat retry spacing is recorded. + if not await self._hmwa_hygiene_cancel_before_commit(fence): + # Bounded overshoot by design: the turn can be held past the budget by up to the + # commit duration. Do NOT "fix" this into a mid-commit cancellation. + _compressed, _ = await attempt.future + return _compressed + fence.release_cancelled_compression_lock() + self._hmwa_hygiene_defer_cleanup(attempt, "session hygiene turn-hold") + # Short flat retry-after: without it every turn re-spawns a compressor, holds it for + # the budget and cancels it — token burn that never commits. _record_hygiene_cooldown( - self, session_entry.session_id, - _HYGIENE_TURNHOLD_RETRY_SECONDS, - "hygiene compression deferred: " - "turn-hold budget expired while the " + self, session_entry.session_id, _HYGIENE_TURNHOLD_RETRY_SECONDS, + "hygiene compression deferred: turn-hold budget expired while the " "summary was still streaming", ) - from agent.session_activity import ( - ActivityProvenance, - ) - _stamp_hygiene_compression_provenance( - _hyg_agent, - "session hygiene compression turn-hold", - ActivityProvenance.AGENT_COMPRESSION_TURNHOLD, - "hygiene compression turn-hold " - "activity stamp failed", - ) - logger.info( - "Session hygiene compression for session %s " - "exceeded turn-hold budget (%.1fs); " - "proceeding without compression this turn", - session_entry.session_id, - time.monotonic() - _hyg_wait_started, - ) - _turnhold_msg = t( - "gateway.compress.turnhold_deferred" - ) - try: - _adapter = self._adapter_for_source(source) - if _adapter and source.chat_id: - await _adapter.send( - source.chat_id, - _turnhold_msg, - metadata=_hyg_meta, - ) - except Exception as _werr: - logger.warning( - "Failed to deliver compression-turnhold " - "notice to user: %s", - _werr, - ) - raise - return _compressed + _log_suffix = "" + self._hmwa_hygiene_stamp( + attempt.agent, "session hygiene compression turn-hold", + "AGENT_COMPRESSION_TURNHOLD", "hygiene compression turn-hold activity stamp failed", + ) + logger.info( + "Session hygiene compression for session %s exceeded turn-hold budget (%.1fs); " + "proceeding without compression this turn%s", + session_entry.session_id, time.monotonic() - attempt.wait_started, _log_suffix, + ) + await self._hmwa_hygiene_notify( + source, attempt.meta, t("gateway.compress.turnhold_deferred"), "compression-turnhold notice", + ) + raise async def _hmwa_hygiene_on_timeout(self, attempt, hs, session_entry, session_key, source): """``except asyncio.TimeoutError`` body: cancel at the commit fence, record the failure cooldown, warn the user, and re-raise; returns the compressed transcript only when the worker crossed the commit boundary first.""" - from gateway.run import ( - _hygiene_compression_timeout_message, - _hygiene_cooldown_for_failure, - _record_hygiene_cooldown, - _stamp_hygiene_compression_provenance, - ) - _hyg_agent = attempt.agent - _hyg_meta = attempt.meta - _hyg_commit_fence = attempt.commit_fence - _hyg_future = attempt.future - _hyg_wait_started = attempt.wait_started - _hyg_timeout_seconds = hs.timeout_seconds - _hyg_total_ceiling_seconds = hs.total_ceiling_seconds - _hyg_failure_cooldown_seconds = hs.failure_cooldown_seconds - - _hyg_waited = time.monotonic() - _hyg_wait_started - _hyg_total_exhausted = ( - _hyg_waited >= _hyg_total_ceiling_seconds - or _hyg_commit_fence.deadline_exceeded - ) + from gateway.run import _hygiene_compression_timeout_message + fence = attempt.commit_fence + _hyg_waited = time.monotonic() - attempt.wait_started + _hyg_total_exhausted = _hyg_waited >= hs.total_ceiling_seconds or fence.deadline_exceeded if _hyg_total_exhausted: - # The worker cooperatively checks this deadline between - # digest calls. Keep its lease until it exits so an - # unchanged session cannot overlap a retry. - _hyg_commit_fence.retain_compression_lock_until_worker_done() - # Capture fence state BEFORE try_cancel — that call itself - # sets is_cancelled, which would mis-label a genuine idle - # timeout as a fence cancel. - _hyg_fence_cancelled = ( - _hyg_commit_fence.is_cancelled + # The worker cooperatively checks this deadline between digest calls. Keep its lease + # until it exits so an unchanged session cannot overlap a retry. + fence.retain_compression_lock_until_worker_done() + # Capture fence state BEFORE try_cancel — that call itself sets is_cancelled, which would + # mis-label a genuine idle timeout as a fence cancel. + _hyg_fence_cancelled = fence.is_cancelled + if not await self._hmwa_hygiene_cancel_before_commit(fence): + # The worker crossed the commit boundary just before the timeout: consume the result + # instead of treating a successful compaction as a timeout. + _compressed, _ = await attempt.future + return _compressed + # Release an inactivity-timed-out worker's holder-qualified lease promptly (no-op for + # total-ceiling attempts, which retained it above). + fence.release_cancelled_compression_lock() + self._hmwa_hygiene_defer_cleanup(attempt, "session hygiene timeout") + await self._hmwa_hygiene_record_failure_cooldown( + hs, session_key, session_entry.session_id, + "session hygiene compression " + ( + "cancelled at commit fence" if _hyg_fence_cancelled + else "total ceiling exhausted" if _hyg_total_exhausted + else "timed out with no output from the summary model" + ), ) - _cancelled = None - while _cancelled is None: - # #76354 F1: a hung commit retains the fence lock; the - # lock-free phase marker keeps this loop from spinning - # forever while the commit blocks. - if _hyg_commit_fence.commit_in_flight: - _cancelled = False - break - _cancelled = ( - _hyg_commit_fence.try_cancel_before_commit() + self._hmwa_hygiene_stamp( + attempt.agent, + "session hygiene compression cancelled at commit fence" if _hyg_fence_cancelled + else "session hygiene compression timed out", + "AGENT_COMPRESSION_TIMEOUT", "hygiene compression timeout activity stamp failed", + ) + if _hyg_fence_cancelled: + logger.warning( + "Session hygiene compression for session %s was cancelled at the " + "commit fence; continuing without compression", session_entry.session_id, ) - if _cancelled is None: - # Round-2 #5: transient lock-setup windows ride - # write patience for seconds; 25ms keeps sub-tick - # latency without 1kHz spin. - await asyncio.sleep(0.025) - if not _cancelled: - # The worker crossed the commit boundary just before the - # timeout; the fence poll waited for it to finish, so - # consume the result instead of treating a successful - # compaction as a timeout. - _compressed, _ = await _hyg_future - else: - # Release an inactivity-timed-out worker's holder- - # qualified lease promptly. Total-ceiling attempts - # retained it above, so this is a no-op for them. - _hyg_commit_fence.release_cancelled_compression_lock() - self._defer_agent_cleanup_until_future_done( - _hyg_future, - _hyg_agent, - context="session hygiene timeout", - ) - attempt.cleanup_deferred = True - _hyg_timeout_error = ( - "session hygiene compression " - "cancelled at commit fence" - if _hyg_fence_cancelled - else ( - "session hygiene compression " - "timed out with no output from " - "the summary model" - ) - ) - if _hyg_failure_cooldown_seconds >= 0: - _hyg_cooldown = await asyncio.to_thread( - _hygiene_cooldown_for_failure, - self, - session_key, - _hyg_failure_cooldown_seconds, - ) - _timeout_reason = ( - _hyg_timeout_error - if _hyg_fence_cancelled - else ( - "session hygiene compression total " - "ceiling exhausted" - if _hyg_total_exhausted - else "session hygiene compression " - "timed out with no output from the " - "summary model" - ) - ) - _record_hygiene_cooldown( - self, session_entry.session_id, - _hyg_cooldown, - _timeout_reason, - ) - from agent.session_activity import ( - ActivityProvenance, - ) - _stamp_hygiene_compression_provenance( - _hyg_agent, - ( - "session hygiene compression " - "cancelled at commit fence" - if _hyg_fence_cancelled - else "session hygiene compression timed out" - ), - ActivityProvenance.AGENT_COMPRESSION_TIMEOUT, - "hygiene compression timeout " - "activity stamp failed", - ) - if _hyg_fence_cancelled: - logger.warning( - "Session hygiene compression for " - "session %s was cancelled at the " - "commit fence; continuing without " - "compression", - session_entry.session_id, - ) - else: - _hyg_elapsed = ( - time.monotonic() - _hyg_wait_started - ) - if _hyg_total_exhausted: - logger.warning( - "Session hygiene compression for session %s " - "reached its total ceiling after %.1fs " - "(progress observed=%s); continuing without " - "compression", - session_entry.session_id, - _hyg_elapsed, - _hyg_commit_fence.progress_observed, - ) - else: - logger.warning( - "Session hygiene compression for session %s " - "made no progress for %.1fs (total wait " - "%.1fs, ceiling %.1fs); continuing without " - "compression", - session_entry.session_id, - _hyg_commit_fence.seconds_since_progress(), - _hyg_elapsed, - _hyg_total_ceiling_seconds, - ) - _timeout_msg = ( - _hygiene_compression_timeout_message( - total_exhausted=_hyg_total_exhausted, - elapsed=_hyg_elapsed, - idle_timeout=_hyg_timeout_seconds, - progress_observed=( - _hyg_commit_fence.progress_observed - ), - ) - ) - try: - _adapter = self._adapter_for_source(source) - if _adapter and source.chat_id: - await _adapter.send( - source.chat_id, - _timeout_msg, - metadata=_hyg_meta, - ) - except Exception as _werr: - logger.warning( - "Failed to deliver compression-timeout " - "warning to user: %s", - _werr, - ) raise - return _compressed + _hyg_elapsed = time.monotonic() - attempt.wait_started + if _hyg_total_exhausted: + logger.warning( + "Session hygiene compression for session %s reached its total ceiling after " + "%.1fs (progress observed=%s); continuing without compression", + session_entry.session_id, _hyg_elapsed, fence.progress_observed, + ) + else: + logger.warning( + "Session hygiene compression for session %s made no progress for %.1fs " + "(total wait %.1fs, ceiling %.1fs); continuing without compression", + session_entry.session_id, fence.seconds_since_progress(), _hyg_elapsed, + hs.total_ceiling_seconds, + ) + await self._hmwa_hygiene_notify( + source, attempt.meta, + _hygiene_compression_timeout_message( + total_exhausted=_hyg_total_exhausted, elapsed=_hyg_elapsed, + idle_timeout=hs.timeout_seconds, progress_observed=fence.progress_observed, + ), + "compression-timeout warning", + ) + raise def _hmwa_hygiene_on_unwind(self, attempt, hs, session_entry, session_key): """``except BaseException`` body (caller re-raises): revoke commit admission and record a - cooldown so the next turn does not immediately re-arm hygiene.""" - from gateway.run import _hygiene_cooldown_for_failure, _record_hygiene_cooldown - _hyg_agent = attempt.agent - _hyg_commit_fence = attempt.commit_fence - _hyg_future = attempt.future - _hyg_failure_cooldown_seconds = hs.failure_cooldown_seconds + cooldown so the next turn does not immediately re-arm hygiene. - # #76354 F2: non-timeout unwind while the detached hygiene - # worker may still run — KeyboardInterrupt, task - # cancellation, or any unexpected error. Revoke commit - # admission (and release the worker's durable lease) BEFORE - # the host unwinds so the worker can never commit later. - _hyg_commit_fence.revoke_commit_admission() + Non-timeout unwind (KeyboardInterrupt, task cancel, unexpected error) while the detached + worker may still run: revoke admission (and release its durable lease) BEFORE the host + unwinds so it can never commit later. Restart drain / task cancel must record a cooldown, + or the next turn re-arms hygiene and waits up to 600s behind a fence that refuses again.""" + from gateway.run import _hygiene_cooldown_for_failure, _record_hygiene_cooldown + attempt.commit_fence.revoke_commit_admission() if not attempt.cleanup_deferred: - self._defer_agent_cleanup_until_future_done( - _hyg_future, - _hyg_agent, - context="session hygiene unwind", - ) - attempt.cleanup_deferred = True - # restart drain / task cancel must record a cooldown, or the - # next turn immediately re-arms hygiene and waits up to 600s - # behind a fence that would refuse the commit again. - if _hyg_failure_cooldown_seconds >= 0: + self._hmwa_hygiene_defer_cleanup(attempt, "session hygiene unwind") + if hs.failure_cooldown_seconds >= 0: try: - _hyg_cooldown = _hygiene_cooldown_for_failure( - self, - session_key, - _hyg_failure_cooldown_seconds, - ) + _hyg_cooldown = _hygiene_cooldown_for_failure(self, session_key, hs.failure_cooldown_seconds) _record_hygiene_cooldown( - self, session_entry.session_id, - _hyg_cooldown, - "session hygiene compression " - "cancelled at commit fence", + self, session_entry.session_id, _hyg_cooldown, + "session hygiene compression cancelled at commit fence", ) except Exception as _cd_err: - logger.debug( - "hygiene unwind cooldown " - "record failed: %s", - _cd_err, - ) + logger.debug("hygiene unwind cooldown record failed: %s", _cd_err) async def _hmwa_hygiene_apply_result( - self, attempt, hs, _compressed, history, *, - _approx_tokens, _msg_count, _warn_token_threshold, + self, attempt, hs, _compressed, history, plan, *, session_entry, session_key, source, _quick_key, run_generation, ): """Adopt a finished hygiene compression (rotation / in-place / refused), rebind the session + turn lease, record streak/cooldown, and warn the user on abort. Publishes the (possibly replaced) transcript on ``attempt.history``.""" - from gateway.run import ( - _hygiene_cooldown_for_failure, - _record_hygiene_cooldown, - _reset_hygiene_failure_streak, - _stamp_hygiene_compression_provenance, - hygiene_compaction_recovered, - ) + from gateway.run import _reset_hygiene_failure_streak, hygiene_compaction_recovered from agent.model_metadata import estimate_messages_tokens_rough _hyg_agent = attempt.agent - _hyg_meta = attempt.meta - _hyg_commit_fence = attempt.commit_fence - _hyg_failure_cooldown_seconds = hs.failure_cooldown_seconds - - # _compress_context ends the old session and creates a new - # session_id. Write compressed messages into the NEW session so - # the old transcript stays intact and searchable. + # _compress_context ends the old session and creates a new session_id; compressed messages + # go into the NEW session so the old transcript stays intact and searchable. _hyg_new_sid = _hyg_agent.session_id _hyg_rotated = _hyg_new_sid != session_entry.session_id - _hyg_in_place = bool( - getattr(_hyg_agent, "_last_compaction_in_place", False) - ) - # Anti-growth guard: refuse a compression that did not shrink - # the transcript (observed: 427K -> 598K). Compare like-for-like - # rough estimates. + _hyg_in_place = bool(getattr(_hyg_agent, "_last_compaction_in_place", False)) + # Anti-growth guard: refuse a compression that did not shrink the transcript (observed: + # 427K -> 598K). Compare like-for-like rough estimates. _hyg_in_toks = estimate_messages_tokens_rough(history) _hyg_out_toks = estimate_messages_tokens_rough(_compressed) if _hyg_rotated and _hyg_out_toks > _hyg_in_toks: @@ -1504,554 +994,377 @@ class GatewayTurnMixin: "Gateway hygiene compression for session %s " "would grow transcript (~%s -> ~%s tokens); " "keeping the original transcript unchanged", - session_entry.session_id, - f"{_hyg_in_toks:,}", - f"{_hyg_out_toks:,}", + session_entry.session_id, f"{_hyg_in_toks:,}", f"{_hyg_out_toks:,}", ) _hyg_rotated = False _compressed = history - # Rewrite the transcript only when rotation produced a NEW - # session id. In-place compaction needs none: - # archive_and_compact() already soft-archived the previous - # active rows and inserted the compacted set, and - # rewrite_transcript() would run - # replace_messages(active_only=False) and DELETE the archived - # turns. Likewise a summary with neither rotation nor a - # completed archive_and_compact() (unchanged session_id) signals - # FAILURE; an unconditional rewrite would replace the originals - # with only the summary (permanent data loss). - # Write-before-repoint (mirrors manual /compress): if - # session_entry were repointed to the child SID and - # rewrite_transcript then failed (lock/ENOSPC), the live entry - # would reference an empty session — the conversation silently - # vanishes. Persist the child transcript first, then rebind. + # Rewrite the transcript only when rotation produced a NEW session id. In-place compaction + # needs none: archive_and_compact() already soft-archived the previous rows, and + # rewrite_transcript() would replace_messages(active_only=False) and DELETE the archived + # turns. A summary with neither rotation nor a completed archive_and_compact() signals + # FAILURE; an unconditional rewrite would replace the originals with only the summary. + # Write-before-repoint (mirrors manual /compress): if session_entry were repointed first + # and rewrite_transcript then failed (lock/ENOSPC), the live entry would reference an + # empty session and the conversation silently vanishes. if _hyg_rotated: - if not await self.async_session_store.rewrite_transcript( - _hyg_new_sid, _compressed - ): + if not await self.async_session_store.rewrite_transcript(_hyg_new_sid, _compressed): logger.error( "Session hygiene: failed to persist " "compressed transcript for rotated " "session %s → %s; keeping the live " "entry on the original session so the " - "conversation is not dropped", - session_entry.session_id, - _hyg_new_sid, + "conversation is not dropped", session_entry.session_id, _hyg_new_sid, ) # Fail closed: treat like no rotation. _hyg_rotated = False _hyg_in_place = False else: session_entry.session_id = _hyg_new_sid - # The held turn lease follows the rotation so an alias - # key resolving the fresh child still serializes against - # this turn. - self._rebind_turn_lease( - _quick_key, run_generation, _hyg_new_sid - ) + # The held turn lease follows the rotation so an alias key resolving the fresh + # child still serializes against this turn. + self._rebind_turn_lease(_quick_key, run_generation, _hyg_new_sid) await self.async_session_store._save() await asyncio.to_thread( - self._sync_telegram_topic_binding, - source, session_entry, - reason="hygiene-compression", + self._sync_telegram_topic_binding, source, session_entry, reason="hygiene-compression", ) - if _hyg_rotated: - # Reset stored token count — transcript rewritten + if _hyg_rotated or _hyg_in_place: + # Transcript rewritten (rotation) or already persisted by archive_and_compact() + # (in-place): reset the stored token count to match the new active set. session_entry.last_prompt_tokens = 0 attempt.history = _compressed _new_count = len(_compressed) - _new_tokens = estimate_messages_tokens_rough( - _compressed - ) - elif _hyg_in_place: - # archive_and_compact() already persisted the - # compacted transcript inside _compress_context. - # Reset counts to match the new active set. - session_entry.last_prompt_tokens = 0 - attempt.history = _compressed - _new_count = len(_compressed) - _new_tokens = estimate_messages_tokens_rough( - _compressed - ) + _new_tokens = estimate_messages_tokens_rough(_compressed) else: - # No rewrite happened — the transcript is unchanged, so the - # post-compression counts equal the pre-compression ones. - _new_count = _msg_count - _new_tokens = _approx_tokens + # No rewrite happened — post-compression counts equal the pre-compression ones. + _new_count = plan.msg_count + _new_tokens = plan.approx_tokens logger.warning( "Gateway hygiene compression for session %s " "did not rotate or compact in place " "(no session_db on the hygiene agent) — " "preserving the original transcript instead " - "of overwriting it with the summary (#21301).", - session_entry.session_id, + "of overwriting it with the summary (#21301).", session_entry.session_id, ) logger.info( "Session hygiene: compressed %s → %s msgs, " "~%s → ~%s tokens", - _msg_count, _new_count, - f"{_approx_tokens:,}", f"{_new_tokens:,}", + plan.msg_count, _new_count, f"{plan.approx_tokens:,}", f"{_new_tokens:,}", ) + if _new_tokens >= plan.warn_token_threshold: + logger.warning("Session hygiene: still ~%s tokens after compression", f"{_new_tokens:,}") - if _new_tokens >= _warn_token_threshold: - logger.warning( - "Session hygiene: still ~%s tokens after " - "compression", - f"{_new_tokens:,}", - ) - - # Summary failure aborts the compressor entirely (messages - # unchanged, nothing dropped). Warn the gateway user visibly - # — agent.log is invisible on TG/Discord/etc. — so they know - # the chat is "frozen" at this size and can /compress to - # retry or /reset to start fresh. + # Summary failure aborts the compressor entirely (messages unchanged, nothing dropped). + # Warn the gateway user visibly — agent.log is invisible on TG/Discord/etc. — so they + # know the chat is "frozen" at this size and can /compress to retry or /reset. _comp = getattr(_hyg_agent, "context_compressor", None) - _hyg_aborted = _comp is not None and getattr( - _comp, "_last_compress_aborted", False - ) - # Fence-cancelled _compress_context returns the original - # transcript with _last_compress_aborted still False - # (failure_class=commit_fence_cancelled, chunk_count=0). Treat - # that no-op as an abort so hygiene records a cooldown instead - # of retrying into the 600s wait. A successful rotate/in-place - # commit is not an abort even if a later invalidation flipped - # the fence. - _hyg_fence_cancelled = bool( - _hyg_commit_fence.is_cancelled - and not _hyg_rotated - and not _hyg_in_place - ) + _hyg_aborted = _comp is not None and getattr(_comp, "_last_compress_aborted", False) + # A fence-cancelled _compress_context returns the original transcript with + # _last_compress_aborted still False. Treat that no-op as an abort so hygiene records a + # cooldown instead of retrying into the 600s wait. A successful rotate/in-place commit is + # not an abort even if a later invalidation flipped the fence. + _hyg_fence_cancelled = bool(attempt.commit_fence.is_cancelled and not _hyg_rotated and not _hyg_in_place) if _hyg_fence_cancelled: _hyg_aborted = True if not _hyg_aborted: - # Recovery decision lives in the unit-tested predicate: the - # degenerate "neither rotated nor compacted in place" path - # sets both flags False and reuses the pre-compression - # counts, so a numbers-only check would read a no-op as - # success and clear the streak. + # Recovery decision lives in the unit-tested predicate: the degenerate "neither + # rotated nor compacted in place" path reuses the pre-compression counts, so a + # numbers-only check would read a no-op as success and clear the streak. if hygiene_compaction_recovered( - aborted=_hyg_aborted, - rotated=_hyg_rotated, - in_place=_hyg_in_place, - msg_count=_msg_count, - new_count=_new_count, - approx_tokens=_approx_tokens, + aborted=_hyg_aborted, rotated=_hyg_rotated, in_place=_hyg_in_place, + msg_count=plan.msg_count, new_count=_new_count, approx_tokens=plan.approx_tokens, new_tokens=_new_tokens, ): - await asyncio.to_thread( - _reset_hygiene_failure_streak, - self, - session_key, - ) + await asyncio.to_thread(_reset_hygiene_failure_streak, self, session_key) if _hyg_aborted: - if _hyg_failure_cooldown_seconds >= 0: - _hyg_cooldown = await asyncio.to_thread( - _hygiene_cooldown_for_failure, - self, - session_key, - _hyg_failure_cooldown_seconds, - ) - _record_hygiene_cooldown( - self, session_entry.session_id, - _hyg_cooldown, - ( - "session hygiene compression " - "cancelled at commit fence" - if _hyg_fence_cancelled - else getattr( - _comp, "_last_summary_error", None - ) - ), - ) - from agent.session_activity import ( - ActivityProvenance, + await self._hmwa_hygiene_record_failure_cooldown( + hs, session_key, session_entry.session_id, + "session hygiene compression cancelled at commit fence" if _hyg_fence_cancelled + else getattr(_comp, "_last_summary_error", None), ) - _stamp_hygiene_compression_provenance( - _hyg_agent, - "session hygiene compression aborted", - ActivityProvenance.AGENT_COMPRESSION_COOLDOWN, - "hygiene compression abort " - "activity stamp failed", + self._hmwa_hygiene_stamp( + _hyg_agent, "session hygiene compression aborted", + "AGENT_COMPRESSION_COOLDOWN", "hygiene compression abort activity stamp failed", ) if not _hyg_fence_cancelled: _err = getattr(_comp, "_last_summary_error", None) or "unknown error" - # Force-redact: provider exception text may contain - # credentials and this message reaches gateway users. + # Force-redact: provider exception text may contain credentials and this message + # reaches gateway users. from agent.redact import redact_sensitive_text _err = redact_sensitive_text(_err, force=True) - _warn_msg = ( - "⚠️ Context compression aborted " + await self._hmwa_hygiene_notify( + source, attempt.meta, "⚠️ Context compression aborted " f"({_err}). No messages were dropped — " "conversation is unchanged. Run /compress " "to retry, /reset for a clean session, or " "check your auxiliary.compression model " - "configuration." + "configuration.", + "compression-failure warning", ) - try: - _adapter = self._adapter_for_source(source) - if _adapter and source.chat_id: - await _adapter.send(source.chat_id, _warn_msg, metadata=_hyg_meta) - except Exception as _werr: - logger.warning( - "Failed to deliver compression-failure warning to user: %s", - _werr, - ) - # Separately: if the user's CONFIGURED aux model failed and we - # recovered by falling back to the main model, tell them — a - # misconfigured auxiliary.compression.model is something only - # they can fix, and silent recovery would hide it. + # If the CONFIGURED aux model failed and we recovered on the main model, tell the user — + # a misconfigured auxiliary.compression.model is something only they can fix. elif _comp is not None and getattr(_comp, "_last_aux_model_failure_model", None): _aux_model = getattr(_comp, "_last_aux_model_failure_model", "") _aux_err = getattr(_comp, "_last_aux_model_failure_error", None) or "unknown error" - _aux_msg = ( - f"ℹ️ Configured compression model `{_aux_model}` " + await self._hmwa_hygiene_notify( + source, attempt.meta, f"ℹ️ Configured compression model `{_aux_model}` " f"failed ({_aux_err}). Recovered using your main " "model — context is intact — but you may want to " - "check `auxiliary.compression.model` in config.yaml." + "check `auxiliary.compression.model` in config.yaml.", + "aux-model-fallback notice", ) + + async def _hmwa_hygiene_codex_compaction(self, hs, plan, history, session_entry, session_key, _hyg_runtime): + """codex app-server runtime: the real context is the server-side thread, not the transcript + mirror. The detached-agent path would only rewrite the mirror and its finally-eviction + would destroy the live thread (next turn starts blank), so use the cached agent's + thread/compact/start and KEEP it cached.""" + from gateway.run import run_codex_hygiene_compaction + _hyg_codex_auto = "native" + _hyg_comp_cfg = hs.data.get("compression") if isinstance(hs.data, dict) else None + if isinstance(_hyg_comp_cfg, dict): + _hyg_codex_auto = str(_hyg_comp_cfg.get("codex_app_server_auto", "native") or "native") + _hyg_codex_outcome = await run_codex_hygiene_compaction( + self, session_key, session_entry.session_id, auto_mode=_hyg_codex_auto, history=history, + approx_tokens=plan.approx_tokens, timeout_seconds=hs.total_ceiling_seconds, + failure_cooldown_seconds=hs.failure_cooldown_seconds, + ) + logger.info( + "Session hygiene (codex app-server): %s " + "(session=%s, mode=%s, ~%s tokens)", + _hyg_codex_outcome, session_entry.session_id, _hyg_codex_auto, f"{plan.approx_tokens:,}", + ) + + async def _hmwa_hygiene_build_agent(self, _hyg_model, _hyg_runtime, session_entry): + """Build the detached hygiene ``AIAgent`` with the live session's system prompt. Returns + ``(agent, sync_session_db)``.""" + from gateway.run import _GATEWAY_HYGIENE_PLATFORM, _seed_hygiene_system_prompt + from run_agent import AIAgent + try: + _hyg_session_row = await self._session_db.get_session(session_entry.session_id) + except Exception as exc: + _hyg_session_row = None + logger.warning( + "Session hygiene could not restore the system " + "prompt for session %s: %s. Preserving an empty " + "prompt so the live turn rebuilds it with its " + "configured providers.", session_entry.session_id, exc, exc_info=True, + ) + _hyg_session_db = getattr(self._session_db, "_db", self._session_db) + # Hygiene is the same lossy rewrite as normal compression: with + # compression.checkpoint_required on, load the memory provider so the checkpoint exists + # before any mutation; otherwise keep the fast path (no provider init). + from hermes_cli.config import load_config as _load_cfg + from utils import is_truthy_value as _is_truthy + + _hyg_checkpoint_required = _is_truthy( + ((_load_cfg() or {}).get("compression") or {}).get("checkpoint_required"), default=False, + ) + _hyg_agent = AIAgent( + **_hyg_runtime, model=_hyg_model, max_iterations=4, quiet_mode=True, + skip_memory=not _hyg_checkpoint_required, enabled_toolsets=["memory"], + session_id=session_entry.session_id, session_db=_hyg_session_db, + ) + _seed_hygiene_system_prompt(_hyg_agent, _hyg_session_row) + # If compression must rebuild instead of retaining the cached prompt, make the persisted + # result deliberately stale for every real gateway surface. + _hyg_agent.platform = _GATEWAY_HYGIENE_PLATFORM + return _hyg_agent, _hyg_session_db + + async def _hmwa_hygiene_detached_attempt( + self, attempt, hs, plan, history, _hyg_msgs, _hyg_model, _hyg_runtime, + source, session_entry, session_key, _quick_key, run_generation, + ): + """Run one detached hygiene compression attempt end to end; publishes the transcript to + continue with (compressed or original) on ``attempt.history``.""" + from gateway.run import HygieneTurnHoldExceeded + from agent.conversation_compression import CompressionCommitFence + _hyg_agent, _hyg_session_db = await self._hmwa_hygiene_build_agent(_hyg_model, _hyg_runtime, session_entry) + attempt.agent = _hyg_agent + try: + # Hygiene runs before the turn and owns the session binding, so prefer in-place + # compaction: archive old rows under the same session id rather than minting a + # continuation child that must be published back to SessionStore/topic bindings. + # Without a SessionDB this stays False and the apply guard preserves it. + _hyg_agent.compression_in_place = True + _bind_hyg_state = getattr(getattr(_hyg_agent, "context_compressor", None), "bind_session_state", None) + if callable(_bind_hyg_state): + _bind_hyg_state(_hyg_session_db, session_entry.session_id) + # Never finalize on close() — that would end the live gateway session row. + _hyg_agent._end_session_on_close = False + _hyg_agent._print_fn = lambda *a, **kw: None + + loop = asyncio.get_running_loop() + _hyg_commit_fence = CompressionCommitFence(total_ceiling_seconds=hs.total_ceiling_seconds) + # Default executor (NOT self._get_executor): a fence-cancelled hung summary must never + # occupy an agent-work slot. But it MUST run in the caller's contextvars: under + # multiplex_profiles the secret scope / HERMES_HOME live in ContextVars, and an empty + # Context makes get_secret() fail closed → lossy truncation. + attempt.commit_fence = _hyg_commit_fence + attempt.future = loop.run_in_executor( + None, + copy_context().run, + lambda: _hyg_agent._compress_context( + _hyg_msgs, "", approx_tokens=plan.approx_tokens, commit_fence=_hyg_commit_fence, + ), + ) + attempt.wait_started = time.monotonic() try: - _adapter = self._adapter_for_source(source) - if _adapter and source.chat_id: - await _adapter.send(source.chat_id, _aux_msg, metadata=_hyg_meta) - except Exception as _werr: - logger.warning( - "Failed to deliver aux-model-fallback notice to user: %s", - _werr, + _compressed = await self._hmwa_hygiene_wait_for_summary(attempt, hs, session_entry) + except HygieneTurnHoldExceeded: + _compressed = await self._hmwa_hygiene_on_turn_hold( + attempt, hs, session_entry, session_key, source, ) + except asyncio.TimeoutError: + _compressed = await self._hmwa_hygiene_on_timeout( + attempt, hs, session_entry, session_key, source, + ) + except BaseException: + self._hmwa_hygiene_on_unwind(attempt, hs, session_entry, session_key) + raise + + await self._hmwa_hygiene_apply_result( + attempt, hs, _compressed, history, plan, session_entry=session_entry, + session_key=session_key, source=source, _quick_key=_quick_key, + run_generation=run_generation, + ) + finally: + # Evict the cached agent so the next turn rebuilds its system prompt from current + # SOUL.md, memory, and skills. + self._evict_cached_agent(session_key) + if not attempt.cleanup_deferred: + await self._cleanup_agent_resources_off_loop(_hyg_agent, context="session hygiene") async def _hmwa_run_session_hygiene( self, event, source, session_entry, session_key, history, _quick_key, run_generation, ): - # Session hygiene: auto-compress pathologically large transcripts before the agent starts so - # oversized histories don't cause repeated truncation/context failures. Token source: the - # API's prompt_tokens from the last turn (session_entry.last_prompt_tokens), else a char/4 - # estimate (30-50% high on code-heavy sessions, so hygiene merely fires a bit early). - from gateway.run import ( - HygieneTurnHoldExceeded, - _GATEWAY_HYGIENE_PLATFORM, - _seed_hygiene_system_prompt, - run_codex_hygiene_compaction, - ) + """Auto-compress pathologically large transcripts before the agent starts so oversized + histories don't cause repeated truncation/context failures. Token source: the API's + prompt_tokens from the last turn, else a char/4 estimate.""" + from gateway.run import HygieneTurnHoldExceeded if not history or len(history) < 4: return history hs = await self._hmwa_hygiene_settings(source, session_key) if not hs.compression_enabled: return history - _needs_compress, _approx_tokens, _msg_count, _warn_token_threshold = ( - await self._hmwa_hygiene_plan(hs, history, session_entry, session_key) - ) - if not _needs_compress: + plan = await self._hmwa_hygiene_plan(hs, history, session_entry, session_key) + if not plan.needs_compress: return history - _hyg_total_ceiling_seconds = hs.total_ceiling_seconds - _hyg_failure_cooldown_seconds = hs.failure_cooldown_seconds - _hyg_data = hs.data - - _hyg_meta = self._thread_metadata_for_source(source, self._reply_anchor_for_event(event)) - + attempt = self._HygieneAttempt( + agent=None, + meta=self._thread_metadata_for_source(source, self._reply_anchor_for_event(event)), + history=history, + ) try: - from agent.conversation_compression import CompressionCommitFence - from run_agent import AIAgent - _hyg_model, _hyg_runtime = self._resolve_session_agent_runtime( - source=source, - session_key=session_key, - user_config=_hyg_data if isinstance(_hyg_data, dict) else None, + source=source, session_key=session_key, + user_config=hs.data if isinstance(hs.data, dict) else None, ) - _hyg_api_mode = str( - _hyg_runtime.get("api_mode") or "" - ).lower() - if _hyg_api_mode == "codex_app_server": - # codex app-server runtime: the real context is the server-side thread, - # not the transcript mirror. The detached-agent block below would only - # rewrite the mirror and its finally-eviction would destroy the live - # thread (next turn starts blank). Use the cached agent's - # thread/compact/start and KEEP it cached. - _hyg_codex_auto = "native" - _hyg_comp_cfg = ( - _hyg_data.get("compression") - if isinstance(_hyg_data, dict) - else None - ) - if isinstance(_hyg_comp_cfg, dict): - _hyg_codex_auto = str( - _hyg_comp_cfg.get( - "codex_app_server_auto", "native" - ) - or "native" - ) - _hyg_codex_outcome = await run_codex_hygiene_compaction( - self, - session_key, - session_entry.session_id, - auto_mode=_hyg_codex_auto, - history=history, - approx_tokens=_approx_tokens, - timeout_seconds=_hyg_total_ceiling_seconds, - failure_cooldown_seconds=_hyg_failure_cooldown_seconds, - ) - logger.info( - "Session hygiene (codex app-server): %s " - "(session=%s, mode=%s, ~%s tokens)", - _hyg_codex_outcome, - session_entry.session_id, - _hyg_codex_auto, - f"{_approx_tokens:,}", + if str(_hyg_runtime.get("api_mode") or "").lower() == "codex_app_server": + await self._hmwa_hygiene_codex_compaction( + hs, plan, history, session_entry, session_key, _hyg_runtime, ) elif _hyg_runtime.get("api_key"): - # Pass the FULL transcript (tool results included), matching the agent - # loop: filtering to user/assistant starved the compressor — tool results - # are the bulk of context and short histories tripped the - # protect-first/last early-return so nothing compressed. - _hyg_msgs = [ - m for m in history - if m.get("role") in {"user", "assistant", "tool"} - ] - + # Pass the FULL transcript (tool results included), matching the agent loop: + # filtering to user/assistant starved the compressor — tool results are the bulk + # of context and short histories tripped the protect-first/last early-return. + _hyg_msgs = [m for m in history if m.get("role") in {"user", "assistant", "tool"}] if len(_hyg_msgs) >= 4: - try: - _hyg_session_row = await self._session_db.get_session( - session_entry.session_id - ) - except Exception as exc: - _hyg_session_row = None - logger.warning( - "Session hygiene could not restore the system " - "prompt for session %s: %s. Preserving an empty " - "prompt so the live turn rebuilds it with its " - "configured providers.", - session_entry.session_id, - exc, - exc_info=True, - ) - _hyg_session_db = getattr(self._session_db, "_db", self._session_db) - # Hygiene is the same lossy rewrite as normal compression: when - # compression.checkpoint_required is on, load the memory provider so - # the checkpoint exists before any mutation; otherwise keep the - # fast path (no provider init, no best-effort hook). - from hermes_cli.config import load_config as _load_cfg - from utils import is_truthy_value as _is_truthy - - _hyg_checkpoint_required = _is_truthy( - ((_load_cfg() or {}).get("compression") or {}).get( - "checkpoint_required" - ), - default=False, + await self._hmwa_hygiene_detached_attempt( + attempt, hs, plan, history, _hyg_msgs, _hyg_model, _hyg_runtime, + source, session_entry, session_key, _quick_key, run_generation, ) - _hyg_agent = AIAgent( - **_hyg_runtime, - model=_hyg_model, - max_iterations=4, - quiet_mode=True, - skip_memory=not _hyg_checkpoint_required, - enabled_toolsets=["memory"], - session_id=session_entry.session_id, - session_db=_hyg_session_db, - ) - _seed_hygiene_system_prompt( - _hyg_agent, - _hyg_session_row, - ) - # If compression must rebuild instead of retaining - # the cached prompt, make the persisted result - # deliberately stale for every real gateway surface. - _hyg_agent.platform = _GATEWAY_HYGIENE_PLATFORM - attempt = self._HygieneAttempt(agent=_hyg_agent, meta=_hyg_meta, history=history) - try: - # Hygiene runs before the turn and owns the session binding, so - # prefer in-place compaction: archive old rows under the same - # session id rather than minting a continuation child that must - # be published back to SessionStore/topic bindings. Without a - # SessionDB this stays False and the guard below preserves it. - _hyg_agent.compression_in_place = True - _bind_hyg_state = getattr( - getattr(_hyg_agent, "context_compressor", None), - "bind_session_state", - None, - ) - if callable(_bind_hyg_state): - _bind_hyg_state( - _hyg_session_db, - session_entry.session_id, - ) - # It must never finalize on close() — close() - # would end the live gateway session row. - _hyg_agent._end_session_on_close = False - _hyg_agent._print_fn = lambda *a, **kw: None - - loop = asyncio.get_running_loop() - _hyg_commit_fence = CompressionCommitFence( - total_ceiling_seconds=_hyg_total_ceiling_seconds - ) - # Default executor (NOT self._get_executor): a fence-cancelled - # hung summary must never occupy an agent-work slot. But it MUST - # run in the caller's contextvars: under multiplex_profiles the - # secret scope / HERMES_HOME live in ContextVars, and an empty - # Context makes get_secret() fail closed → lossy truncation. - _hyg_future = loop.run_in_executor( - None, - copy_context().run, - lambda: _hyg_agent._compress_context( - _hyg_msgs, "", - approx_tokens=_approx_tokens, - commit_fence=_hyg_commit_fence, - ), - ) - attempt.commit_fence = _hyg_commit_fence - attempt.future = _hyg_future - attempt.wait_started = time.monotonic() - try: - _compressed = await self._hmwa_hygiene_wait_for_summary( - attempt, hs, session_entry, - ) - except HygieneTurnHoldExceeded: - _compressed = await self._hmwa_hygiene_on_turn_hold( - attempt, hs, session_entry, session_key, source, - ) - except asyncio.TimeoutError: - _compressed = await self._hmwa_hygiene_on_timeout( - attempt, hs, session_entry, session_key, source, - ) - except BaseException: - self._hmwa_hygiene_on_unwind(attempt, hs, session_entry, session_key) - raise - - await self._hmwa_hygiene_apply_result( - attempt, hs, _compressed, history, - _approx_tokens=_approx_tokens, - _msg_count=_msg_count, - _warn_token_threshold=_warn_token_threshold, - session_entry=session_entry, - session_key=session_key, - source=source, - _quick_key=_quick_key, - run_generation=run_generation, - ) - finally: - history = attempt.history - # Evict the cached agent so the next turn rebuilds its system - # prompt from current SOUL.md, memory, and skills. - self._evict_cached_agent(session_key) - if not attempt.cleanup_deferred: - await self._cleanup_agent_resources_off_loop( - _hyg_agent, context="session hygiene" - ) except HygieneTurnHoldExceeded: - # Availability boundary, not a failure — already logged at INFO by the turn- - # hold handler. Must not hit the generic "auto-compress failed" warning - # below: that log made thinking-model deployments read as permanently broken. + # Availability boundary, not a failure — already logged at INFO by the turn-hold + # handler; the generic warning below made thinking-model deployments read as broken. pass except Exception as e: - logger.warning( - "Session hygiene auto-compress failed: %s", e - ) - return history + logger.warning("Session hygiene auto-compress failed: %s", e) + return attempt.history async def _hmwa_first_contact_notes(self, source, history, turn_sidecar_notes): """First-ever-message onboarding note + one-time 'no home channel' prompt (both only when - the session has no history).""" + the session has no history). Delivered on the user message (sidecar), NOT the ephemeral + system prompt: present-on-turn-1/absent-on-turn-2 was a guaranteed prompt diff + rebuild.""" from gateway.run import _hermes_home, _home_target_env_var, _load_gateway_config - # First-message onboarding -- only on the very first interaction ever. Delivered on the - # current user message (sidecar), NOT the ephemeral system prompt: present-on-turn-1/absent- - # on-turn-2 was a guaranteed system-prompt diff and agent rebuild. - if not history and not await self.async_session_store.has_any_sessions(): - # Default first-contact note: a brief self-introduction. + if history: + return + if not await self.async_session_store.has_any_sessions(): _intro_note = ( "[System note: This is the user's very first message ever. " "Briefly introduce yourself and mention that /help shows available commands. " "Keep the introduction concise -- one or two sentences max.]" ) - # Opt-in structured profile-build path: when enabled (default "ask") and not yet offered - # on this install, swap the plain intro for a consent-gated directive that offers to - # build a user profile and persists confirmed facts via memory(target="user"). Fires at - # most once (onboarding.seen flag); onboarding.profile_build: off in config.yaml - # disables it. + # Opt-in profile-build path: when onboarding.profile_build is "ask" (default) and not + # yet offered on this install, swap the plain intro for a consent-gated directive that + # offers to build a user profile via memory(target="user"). Fires at most once. try: from agent.onboarding import ( - PROFILE_BUILD_FLAG, - is_seen, - mark_seen, - profile_build_directive, + PROFILE_BUILD_FLAG, is_seen, mark_seen, profile_build_directive, profile_build_mode, ) _onb_cfg = _load_gateway_config() - if ( - profile_build_mode(_onb_cfg) == "ask" - and not is_seen(_onb_cfg, PROFILE_BUILD_FLAG) - ): + if profile_build_mode(_onb_cfg) == "ask" and not is_seen(_onb_cfg, PROFILE_BUILD_FLAG): turn_sidecar_notes.append(profile_build_directive().strip()) mark_seen(_hermes_home / "config.yaml", PROFILE_BUILD_FLAG) else: turn_sidecar_notes.append(_intro_note) except Exception as _pb_err: - logger.debug( - "Profile-build onboarding directive failed, using plain intro: %s", - _pb_err, - ) + logger.debug("Profile-build onboarding directive failed, using plain intro: %s", _pb_err) turn_sidecar_notes.append(_intro_note) - # One-time prompt if no home channel is set for this platform - # Skip for webhooks - they deliver directly to configured targets (github_comment, etc.) - if not history and source.platform and source.platform != Platform.LOCAL and source.platform != Platform.WEBHOOK: - platform_name = source.platform.value - env_key = _home_target_env_var(platform_name) - # Multiplex: home channel may live only in the profile secret - # scope / PlatformConfig, not process os.environ. - home_env = "" - try: - from agent.secret_scope import get_secret + # One-time prompt if no home channel is set for this platform. Skipped for webhooks — + # they deliver directly to configured targets (github_comment, etc.). + if not source.platform or source.platform in (Platform.LOCAL, Platform.WEBHOOK): + return + platform_name = source.platform.value + env_key = _home_target_env_var(platform_name) + # Multiplex: the home channel may live only in the profile secret scope / PlatformConfig, + # not process os.environ. + home_env = "" + try: + from agent.secret_scope import get_secret - home_env = (get_secret(env_key) or "").strip() if env_key else "" - except Exception: - home_env = "" - if not home_env: - home_env = (os.getenv(env_key) or "").strip() if env_key else "" - # Also honor in-memory / yaml home_channel on this platform. + home_env = (get_secret(env_key) or "").strip() if env_key else "" + except Exception: + home_env = "" + if not home_env: + home_env = (os.getenv(env_key) or "").strip() if env_key else "" + # Also honor in-memory / yaml home_channel on this platform. + try: + if not home_env and self.config.get_home_channel(source.platform): + home_env = "set" + except Exception: + pass + # Secondary-profile platforms (e.g. Slack on yolo) may only exist under that profile's + # loaded config — re-read live config inside the already-installed scope. + if not home_env: try: - if not home_env and self.config.get_home_channel(source.platform): + from gateway.config import load_gateway_config as _lgc + prof = (getattr(source, "profile", None) or "").strip() + if prof and prof != "default" and _lgc().get_home_channel(source.platform): home_env = "set" except Exception: pass - # Secondary-profile platforms (e.g. Slack on yolo) may only exist - # under that profile's loaded config — check after scope install. - if not home_env: - try: - from gateway.config import load_gateway_config as _lgc - prof = (getattr(source, "profile", None) or "").strip() - if prof and prof != "default": - # Already inside profile scope for secondary handlers; - # re-read live config for home_channel. - _pcfg = _lgc() - if _pcfg.get_home_channel(source.platform): - home_env = "set" - except Exception: - pass - if not home_env: - # Slack dispatches all Hermes commands through a single - # parent slash command `/hermes`; bare `/sethome` is not - # registered and would fail with "app did not respond". - sethome_cmd = ( - "/hermes sethome" - if source.platform == Platform.SLACK - else "/sethome" - ) - notice = ( - f"📬 No home channel is set for {platform_name.title()}. " - f"A home channel is where Hermes delivers cron job results " - f"and cross-platform messages.\n\n" - f"Type {sethome_cmd} to make this chat your home channel, " - f"or ignore to skip." - ) - await self._deliver_platform_notice(source, notice) + if not home_env: + # Slack routes every Hermes command through the single parent slash command + # `/hermes`; bare `/sethome` is not registered and would fail. + sethome_cmd = "/hermes sethome" if source.platform == Platform.SLACK else "/sethome" + await self._deliver_platform_notice( + source, f"📬 No home channel is set for {platform_name.title()}. " + f"A home channel is where Hermes delivers cron job results " + f"and cross-platform messages.\n\n" + f"Type {sethome_cmd} to make this chat your home channel, " + f"or ignore to skip.", + ) def _hmwa_apply_message_timestamp(self, event, message_text): - # Capture the platform event time as message metadata and keep the persisted transcript - # clean (strip any leading timestamp prefix). This runs regardless of the toggle so storage - # stays clean and the send-time is preserved. Only the in-context RENDER (the prefix the - # model sees) is gated behind gateway.message_timestamps.enabled — default OFF. + """Capture the platform event time as message metadata and keep the persisted transcript + clean (strip any leading timestamp prefix) regardless of the toggle; only the in-context + RENDER is gated behind gateway.message_timestamps.enabled (default OFF).""" from gateway.run import _load_gateway_config, _message_timestamps_enabled persist_user_message = None persist_user_timestamp = None @@ -2063,24 +1376,16 @@ class GatewayTurnMixin: strip_leading_message_timestamps as _strip_msg_ts, ) _evt_tz = _get_evt_tz() - _evt_ts = getattr(event, "timestamp", None) if message_text and isinstance(message_text, str): - _clean_message_text, _embedded_ts = _strip_msg_ts( - message_text, tz=_evt_tz) + _clean_message_text, _embedded_ts = _strip_msg_ts(message_text, tz=_evt_tz) persist_user_message = _clean_message_text - _event_epoch = _coerce_msg_ts(_evt_ts, tz=_evt_tz) - persist_user_timestamp = ( - _event_epoch if _event_epoch is not None else _embedded_ts - ) + _event_epoch = _coerce_msg_ts(getattr(event, "timestamp", None), tz=_evt_tz) + persist_user_timestamp = _event_epoch if _event_epoch is not None else _embedded_ts if _message_timestamps_enabled(_load_gateway_config()): - message_text = _render_msg_ts( - _clean_message_text, - persist_user_timestamp, - tz=_evt_tz, - ) + message_text = _render_msg_ts(_clean_message_text, persist_user_timestamp, tz=_evt_tz) else: - # Toggle off: model sees the clean message; the timestamp - # is still stored as metadata for later opt-in. + # Toggle off: the model sees the clean message; the timestamp is still stored + # as metadata for later opt-in. message_text = _clean_message_text except Exception as _ts_err: logger.debug("Message timestamp injection failed (non-fatal): %s", _ts_err) @@ -2091,16 +1396,12 @@ class GatewayTurnMixin: workspace, so preserve the routing metadata used by the response delivery path.""" try: _typing_adapter = self._adapter_for_source(source) - _stop_with_metadata = getattr( - type(_typing_adapter), "_stop_typing_with_metadata", None - ) + _stop_with_metadata = getattr(type(_typing_adapter), "_stop_typing_with_metadata", None) _stop_typing = getattr(type(_typing_adapter), "stop_typing", None) if _typing_adapter and callable(_stop_with_metadata): await _typing_adapter._stop_typing_with_metadata( source.chat_id, - self._thread_metadata_for_source( - source, self._reply_anchor_for_event(event) - ), + self._thread_metadata_for_source(source, self._reply_anchor_for_event(event)), ) elif _typing_adapter and callable(_stop_typing): await _typing_adapter.stop_typing(source.chat_id) @@ -2116,29 +1417,22 @@ class GatewayTurnMixin: post-compression session_id propagation. Returns ``(response, _intentional_silence, agent_messages)``.""" from gateway.run import ( - _is_gateway_hidden_reasoning_incomplete_turn, - _normalize_empty_agent_response, - _sanitize_gateway_final_response, - _should_clear_resume_pending_after_turn, + _is_gateway_hidden_reasoning_incomplete_turn, _normalize_empty_agent_response, + _sanitize_gateway_final_response, _should_clear_resume_pending_after_turn, ) response = agent_result.get("final_response") or "" - # Hidden-reasoning-only retry exhaustion: the loop's sentinel text ("Codex response - # remained incomplete after 3 continuation attempts") doubles as final_response, so it - # would be delivered verbatim into the channel — where peer agents can ingest it as a + # Hidden-reasoning-only retry exhaustion: the loop's sentinel text doubles as + # final_response and would be delivered verbatim — where peer agents can ingest it as a # completed assistant turn. if _is_gateway_hidden_reasoning_incomplete_turn(agent_result): response = "" try: from gateway.response_filters import is_intentional_silence_agent_result - _intentional_silence = is_intentional_silence_agent_result( - agent_result, response, - ) + _intentional_silence = is_intentional_silence_agent_result(agent_result, response) except Exception: _intentional_silence = False - # Convert the agent's internal "(empty)" sentinel into a user-friendly message. - # "(empty)" means the model failed to produce visible content after exhausting all - # retries (nudge, prefill, empty-retry, fallback). + # "(empty)" = the model produced no visible content after exhausting all retries. if response == "(empty)" and not _intentional_silence: response = ( "⚠️ The model returned no response after processing tool " @@ -2146,20 +1440,13 @@ class GatewayTurnMixin: "rephrase your question." ) agent_messages = agent_result.get("messages", []) - _response_time = time.time() - _msg_start_time - _api_calls = agent_result.get("api_calls", 0) - _resp_len = len(response) logger.info( "response ready: platform=%s chat=%s time=%.1fs api_calls=%d response=%d chars", _platform_name, source.chat_id or "unknown", - _response_time, _api_calls, _resp_len, + time.time() - _msg_start_time, agent_result.get("api_calls", 0), len(response), ) - # The cross-process cache-coherence re-baseline (_refresh_agent_cache_message_count) is - # deferred until AFTER the transcript persistence block below: it must include the - # first-turn `session_meta` marker row and the compression session_id swap. - - # Successful turn: clear the stuck-loop counter (it only accumulates across CONSECUTIVE + # Successful turn: clear the stuck-loop counter (accumulates only across CONSECUTIVE # restarts where the session never completed) and resume_pending (set by drain-timeout # shutdown) so later messages don't get the restart-interruption system note. if session_key and _should_clear_resume_pending_after_turn(agent_result): @@ -2167,148 +1454,117 @@ class GatewayTurnMixin: try: await self.async_session_store.clear_resume_pending(session_key) except Exception as _e: - logger.debug( - "clear_resume_pending failed for %s: %s", - session_key, _e, - ) + logger.debug("clear_resume_pending failed for %s: %s", session_key, _e) - # Normalize empty responses: surface errors, partial failures, and - # the case where agent did work but returned no text. Fix for #18765. + # Normalize empty responses: surface errors, partial failures, and work-without-text. if not _intentional_silence: - response = _normalize_empty_agent_response( - agent_result, response, history_len=len(history), - ) + response = _normalize_empty_agent_response(agent_result, response, history_len=len(history)) response = _sanitize_gateway_final_response(source.platform, response) # Ordering contract: the agent thread already updated the contextvar in - # conversation_compression.py; propagate to SessionEntry + _save(). + # conversation_compression.py; propagate to SessionEntry + _save() — but only if the + # binding still points at the session this run was launched against. if agent_result.get("session_id") and agent_result["session_id"] != session_entry.session_id: if session_entry.session_id == _run_start_session_id: session_entry.session_id = agent_result["session_id"] - # The held turn lease follows the rotation: the transcript persistence below - # writes to the NEW id, so the serialization boundary must move with it or an - # alias key resolving the fresh child could interleave. - self._rebind_turn_lease( - _quick_key, run_generation, session_entry.session_id - ) + # The held turn lease follows the rotation: transcript persistence writes to the + # NEW id, so the serialization boundary must move with it. + self._rebind_turn_lease(_quick_key, run_generation, session_entry.session_id) await self.async_session_store._save() await self.async_session_store._record_gateway_session_peer( - session_entry.session_id, - session_key, - source, + session_entry.session_id, session_key, source, ) await asyncio.to_thread( - self._sync_telegram_topic_binding, - source, session_entry, reason="agent-result-compression", + self._sync_telegram_topic_binding, source, session_entry, reason="agent-result-compression", ) else: logger.info( "Skipping agent-result session split sync for %s because " "the session binding moved from %s to %s before " "compression finished", - session_key or "?", - _run_start_session_id, - session_entry.session_id, + session_key or "?", _run_start_session_id, session_entry.session_id, ) return response, _intentional_silence, agent_messages + # reasoning_style → (header line, per-line quote prefix for blank / non-blank lines) + _REASONING_QUOTE_STYLES = { + "subtext": ("-# 💭 Reasoning", "-# ", "-#"), + "blockquote": ("> 💭 **Reasoning:**", "> ", ">"), + } + def _hmwa_prepend_reasoning(self, agent_result, response, source, _intentional_silence): - # Prepend reasoning if display is enabled (per-platform). Mattermost requires explicit - # opt-in because this is scratch text, not ordinary final-answer content. + """Prepend the last reasoning block when show_reasoning is on for this platform. Mattermost + requires an explicit per-platform opt-in (scratch text, not final-answer content).""" from gateway.run import _load_gateway_config, _platform_config_key, _resolve_gateway_display_bool try: _show_reasoning_effective = _resolve_gateway_display_bool( - _load_gateway_config(), - _platform_config_key(source.platform), - "show_reasoning", - default=bool(getattr(self, "_show_reasoning", False)), - platform=source.platform, + _load_gateway_config(), _platform_config_key(source.platform), "show_reasoning", + default=bool(getattr(self, "_show_reasoning", False)), platform=source.platform, require_platform_override_for={Platform.MATTERMOST}, ) except Exception: _show_reasoning_effective = ( - False - if source.platform == Platform.MATTERMOST - else getattr(self, "_show_reasoning", False) + False if source.platform == Platform.MATTERMOST else getattr(self, "_show_reasoning", False) ) - if _show_reasoning_effective and response and not _intentional_silence: - last_reasoning = agent_result.get("last_reasoning") - if last_reasoning: - from gateway.stream_consumer import escape_code_fences_for_display - # Collapse long reasoning to keep messages readable - lines = last_reasoning.strip().splitlines() - if len(lines) > 15: - display_reasoning = "\n".join(lines[:15]) - display_reasoning += f"\n_... ({len(lines) - 15} more lines)_" - else: - display_reasoning = last_reasoning.strip() - # Render style is per-platform: Discord defaults to "-# " subtext (native small - # grey metadata text); other platforms keep the fenced code block. - try: - from gateway.display_config import resolve_display_setting - _reasoning_style = resolve_display_setting( - _load_gateway_config(), - _platform_config_key(source.platform), - "reasoning_style", - "code", - ) - except Exception: - _reasoning_style = "code" - if _reasoning_style == "subtext": - _quoted = "\n".join( - f"-# {ln}" if ln else "-#" for ln in display_reasoning.splitlines() - ) - response = f"-# 💭 Reasoning\n{_quoted}\n\n{response}" - elif _reasoning_style == "blockquote": - _quoted = "\n".join( - f"> {ln}" if ln else ">" for ln in display_reasoning.splitlines() - ) - response = f"> 💭 **Reasoning:**\n{_quoted}\n\n{response}" - else: - # Escape ``` inside reasoning so inner fences don't - # break the outer code block used to render it. - display_reasoning = escape_code_fences_for_display(display_reasoning) - response = f"💭 **Reasoning:**\n```\n{display_reasoning}\n```\n\n{response}" - return response + if not (_show_reasoning_effective and response and not _intentional_silence): + return response + last_reasoning = agent_result.get("last_reasoning") + if not last_reasoning: + return response + from gateway.stream_consumer import escape_code_fences_for_display + # Collapse long reasoning to keep messages readable + lines = last_reasoning.strip().splitlines() + if len(lines) > 15: + display_reasoning = "\n".join(lines[:15]) + f"\n_... ({len(lines) - 15} more lines)_" + else: + display_reasoning = last_reasoning.strip() + # Render style is per-platform: Discord defaults to "-# " subtext (native small grey + # metadata text); other platforms keep the fenced code block. + try: + from gateway.display_config import resolve_display_setting + _reasoning_style = resolve_display_setting( + _load_gateway_config(), _platform_config_key(source.platform), "reasoning_style", "code", + ) + except Exception: + _reasoning_style = "code" + _quote = self._REASONING_QUOTE_STYLES.get(_reasoning_style) + if _quote: + header, prefix, empty = _quote + _quoted = "\n".join(f"{prefix}{ln}" if ln else empty for ln in display_reasoning.splitlines()) + return f"{header}\n{_quoted}\n\n{response}" + # Escape ``` inside reasoning so inner fences don't break the outer code block. + display_reasoning = escape_code_fences_for_display(display_reasoning) + return f"💭 **Reasoning:**\n```\n{display_reasoning}\n```\n\n{response}" def _hmwa_runtime_footer_line(self, agent_result, source, _turn_seconds): - # Runtime-metadata footer — only on the FINAL message of the turn. Off by default - # (display.runtime_footer.enabled=false). When streaming already delivered the body, we - # can't mutate the sent text, so we fire a separate trailing send below. + """Runtime-metadata footer for the FINAL message of the turn; off by default + (display.runtime_footer.enabled=false).""" from gateway.run import _load_gateway_config, _platform_config_key, _terminal_scope_cwd - _footer_line = "" try: from gateway.runtime_footer import build_footer_line as _bfl - _footer_line = _bfl( + return _bfl( user_config=_load_gateway_config(), - platform_key=_platform_config_key(source.platform), - model=agent_result.get("model"), + platform_key=_platform_config_key(source.platform), model=agent_result.get("model"), context_tokens=agent_result.get("last_prompt_tokens", 0) or 0, context_length=agent_result.get("context_length") or None, - cwd=_terminal_scope_cwd(""), - turn_seconds=_turn_seconds, + cwd=_terminal_scope_cwd(""), turn_seconds=_turn_seconds, ) except Exception as _footer_err: logger.debug("runtime_footer build failed: %s", _footer_err) - _footer_line = "" - return _footer_line + return "" async def _hmwa_post_turn_hooks(self, hook_ctx, agent_result, response): """agent:end hook, process-watcher scheduling, and watch-notification drain.""" - # Emit agent:end hook await self.hooks.emit("agent:end", { - **hook_ctx, - "response": (response or "")[:500], - "model": agent_result.get("model", ""), + **hook_ctx, "response": (response or "")[:500], "model": agent_result.get("model", ""), "provider": agent_result.get("provider", ""), }) - # Check for pending process watchers (check_interval on background processes) + # Pending process watchers (check_interval on background processes) try: from tools.process_registry import process_registry - # Detach the current batch atomically (see crash-recovery drain - # above): reassign to a fresh list so a watcher appended by a - # concurrent session during the yield isn't dropped by clear(). + # Detach the current batch atomically: reassign to a fresh list so a watcher appended + # by a concurrent session during the yield isn't dropped by clear(). watchers = process_registry.pending_watchers process_registry.pending_watchers = [] for i, watcher in enumerate(watchers): @@ -2319,42 +1575,40 @@ class GatewayTurnMixin: logger.error("Process watcher setup error: %s", e) # Drain watch notifications that arrived during the run. The queue also carries process - # completions (handled by the per-process watcher task above) and async-delegation - # completions (owned by _async_delegation_watcher, the single consumer for idle and - # post-turn cases) — inject only watch-type events and leave the rest on the queue. + # completions (per-process watcher task above) and async-delegation completions (owned by + # _async_delegation_watcher) — inject only watch-type events, leave the rest queued. try: from tools.process_registry import process_registry as _pr await self._drain_watch_notifications(_pr.completion_queue) except Exception as e: logger.debug("Watch queue drain error: %s", e) + _CONTEXT_OVERFLOW_ERROR_PHRASES = ( + "context length", "context size", "context window", + "maximum context", "token limit", "too many tokens", + "reduce the length", "exceeds the limit", + "request entity too large", "prompt is too long", + "payload too large", "input is too long", + ) + def _hmwa_classify_turn_failure(self, agent_result, history, session_entry): """Classify a finished turn for transcript persistence. Returns - ``(agent_failed_early, hidden_reasoning_incomplete, is_context_overflow_failure)``.""" + ``(agent_failed_early, hidden_reasoning_incomplete, is_context_overflow_failure)``. + + Context-overflow failures (compression exhausted, generic 400 on large sessions) must + NOT persist the user message — it would grow the session and reproduce the failure + forever. Transient failures (429, timeout, connection error, 5xx) DO persist it: the + session is not oversized and dropping the turn causes severe context loss on retry.""" from gateway.run import _is_gateway_hidden_reasoning_incomplete_turn - # Persist the full agent loop (tool calls, results, reasoning) so sessions resume with - # full context. IMPORTANT: on context-overflow failures (compression exhausted, generic - # 400 on large sessions) do NOT persist the user message — it would grow the session and - # reproduce the failure forever. Transient failures (429, timeout, connection error, - # 5xx) are different: the session is not oversized and dropping the user turn causes - # severe context loss on retry, so persist it. agent_failed_early = bool(agent_result.get("failed")) - hidden_reasoning_incomplete = _is_gateway_hidden_reasoning_incomplete_turn( - agent_result - ) - _err_str_for_classify = str(agent_result.get("error", "")).lower() - # Use specific multi-word phrases (not bare "exceed"/"token") to avoid false positives - # on transient errors such as "rate limit exceeded"; matches run_agent.py's classifier. + hidden_reasoning_incomplete = _is_gateway_hidden_reasoning_incomplete_turn(agent_result) + _err = str(agent_result.get("error", "")).lower() + # Specific multi-word phrases (not bare "exceed"/"token") avoid false positives on + # transient errors such as "rate limit exceeded"; matches run_agent.py's classifier. is_context_overflow_failure = agent_failed_early and ( bool(agent_result.get("compression_exhausted")) - or any(p in _err_str_for_classify for p in ( - "context length", "context size", "context window", - "maximum context", "token limit", "too many tokens", - "reduce the length", "exceeds the limit", - "request entity too large", "prompt is too long", - "payload too large", "input is too long", - )) - or ("400" in _err_str_for_classify and len(history) > 50) + or any(p in _err for p in self._CONTEXT_OVERFLOW_ERROR_PHRASES) + or ("400" in _err and len(history) > 50) ) if is_context_overflow_failure: logger.info( @@ -2365,27 +1619,23 @@ class GatewayTurnMixin: elif agent_failed_early: logger.info( "Transient agent failure in session %s — persisting user " - "message so conversation context is preserved on retry.", - session_entry.session_id, + "message so conversation context is preserved on retry.", session_entry.session_id, ) elif hidden_reasoning_incomplete: logger.warning( "Suppressing hidden-reasoning-only incomplete gateway turn " "for session %s: %s", - session_entry.session_id, - agent_result.get("error", "processing incomplete"), + session_entry.session_id, agent_result.get("error", "processing incomplete"), ) return agent_failed_early, hidden_reasoning_incomplete, is_context_overflow_failure async def _hmwa_compression_exhaustion_reset( self, agent_result, response, session_entry, session_key, source, ): - """Auto-reset a permanently oversized session (never on a lock-contended defer). Returns + """Auto-reset a permanently oversized session so the next message starts fresh instead of + replaying the oversized context forever. Never on a lock-contended defer — that is the + OPPOSITE case (a concurrent path holds the lock and is shrinking it). Returns ``(response, session_entry)``.""" - # Compression exhausted = permanently too large: auto-reset so the next message starts - # fresh instead of replaying the oversized context forever. A lock-contended defer is - # the OPPOSITE case (a concurrent path holds the lock and is shrinking it): never wipe - # for that. if agent_result.get("compression_deferred"): logger.info( "Compression deferred for session %s — the compression " @@ -2394,27 +1644,21 @@ class GatewayTurnMixin: session_entry.session_id if session_entry else "?", ) elif agent_result.get("compression_exhausted") and session_entry and session_key: - logger.info( - "Auto-resetting session %s after compression exhaustion.", - session_entry.session_id, - ) + logger.info("Auto-resetting session %s after compression exhaustion.", session_entry.session_id) new_entry = await self.async_session_store.reset_session(session_key) self._evict_cached_agent(session_key) - # Conversation boundary: one funnel call clears every conversation-scoped - # per-session dict (see _CONVERSATION_SCOPED_STATE). - self._clear_conversation_scope( - session_key, reason="compression_exhausted_reset" - ) + # Conversation boundary: one funnel call clears every conversation-scoped per-session + # dict (see _CONVERSATION_SCOPED_STATE). + self._clear_conversation_scope(session_key, reason="compression_exhausted_reset") if new_entry is not None: # Re-point the Telegram topic binding at the fresh session: compression rotated # session_entry.session_id to the bloated child earlier this turn and that _sync - # also rewrote the (chat_id, thread_id) binding. Without a re-sync the - # binding-heal walk switches the next inbound message back onto the child and - # re-triggers exhaustion forever. No-op on non-topic lanes. + # also rewrote the (chat_id, thread_id) binding. Without a re-sync the binding-heal + # walk switches the next inbound message back onto the child and re-triggers + # exhaustion forever. No-op on non-topic lanes. session_entry = new_entry await asyncio.to_thread( - self._sync_telegram_topic_binding, - source, session_entry, reason="compression-exhausted-reset", + self._sync_telegram_topic_binding, source, session_entry, reason="compression-exhausted-reset", ) response = (response or "") + ( "\n\n🔄 Session auto-reset — the conversation exceeded the " @@ -2423,6 +1667,22 @@ class GatewayTurnMixin: ) return response, session_entry + @staticmethod + def _hmwa_user_transcript_entry( + event, message_text, persist_user_message, persist_user_timestamp, persist_user_display_kind, ts, + ): + """Transcript row for the inbound user turn (clean text + event time when captured).""" + _user_entry = { + "role": "user", + "content": persist_user_message if persist_user_message is not None else message_text, + "timestamp": persist_user_timestamp if persist_user_timestamp is not None else ts, + } + if persist_user_display_kind: + _user_entry["display_kind"] = persist_user_display_kind + if getattr(event, "message_id", None): + _user_entry["message_id"] = str(event.message_id) + return _user_entry + async def _hmwa_persist_turn_transcript( self, *, event, source, session_entry, session_key, agent_result, agent_messages, history, response, message_text, persist_user_message, persist_user_timestamp, @@ -2434,121 +1694,70 @@ class GatewayTurnMixin: cached agent's message count.""" from gateway.run import _resolve_gateway_model ts = time.time() # Unix epoch float — consistent with DB storage + store = self.async_session_store + sid = session_entry.session_id + # The agent already persisted this turn's rows via _flush_messages_to_session_db() (the + # codex app-server runtime too: it flushes its own projection and reports + # agent_persisted=True); skip the DB write to avoid duplicates. Default = a session DB + # exists; a non-persisting runtime opts in via False. + agent_persisted = agent_result.get("agent_persisted", self._session_db is not None) + + def _user_entry(): + return self._hmwa_user_transcript_entry( + event, message_text, persist_user_message, persist_user_timestamp, + persist_user_display_kind, ts, + ) - # Fresh session (no history): write the full tool definitions as the first entry so the - # transcript is self-describing — the same dicts sent as tools=[...] in the API request. if is_context_overflow_failure: pass # Skip all transcript writes — don't grow a broken session elif not history: - tool_defs = agent_result.get("tools", []) - await self.async_session_store.append_to_transcript( - session_entry.session_id, - { - "role": "session_meta", - "tools": tool_defs or [], - "model": _resolve_gateway_model(), - "platform": source.platform.value if source.platform else "", - "timestamp": ts, - } - ) + # Fresh session: write the full tool definitions as the first entry so the transcript + # is self-describing — the same dicts sent as tools=[...] in the API request. + await store.append_to_transcript(sid, { + "role": "session_meta", + "tools": agent_result.get("tools", []) or [], + "model": _resolve_gateway_model(), + "platform": source.platform.value if source.platform else "", + "timestamp": ts, + }) - # The agent already persisted these via _flush_messages_to_session_db(); skip the DB - # write to avoid duplicates. Holds for the codex app-server runtime too (it flushes its - # own projected messages before returning and reports agent_persisted=True). Reading the - # flag (default = self._session_db is not None) keeps the contract explicit; a - # non-persisting runtime opts in via False. - agent_persisted = agent_result.get("agent_persisted", self._session_db is not None) - - # Only the NEW messages from this turn: use history_offset (what the agent saw), not - # len(history), which counts session_meta entries stripped before the agent saw them. if is_context_overflow_failure: - pass # handled above — skip all transcript writes + pass elif agent_failed_early or hidden_reasoning_incomplete: # Transient failure (429/timeout/5xx): persist only the user message so the next - # message can load a transcript that reflects what was said. Skip the assistant - # error text since it's a gateway-generated hint, not model output. Hidden-reasoning - # incomplete turns follow the same rule so peer-agent channels don't ingest them. - _user_entry = { - "role": "user", - "content": ( - persist_user_message - if persist_user_message is not None - else message_text - ), - "timestamp": ( - persist_user_timestamp - if persist_user_timestamp is not None - else ts - ), - } - if persist_user_display_kind: - _user_entry["display_kind"] = persist_user_display_kind - if event.message_id: - _user_entry["message_id"] = str(event.message_id) - # Dedupe: skip if this platform message_id is already in the transcript (prevents - # duplicate user turns on Telegram retries after transient failures). - _skip_persist = ( - event.message_id - and await self.async_session_store.has_platform_message_id( - session_entry.session_id, str(event.message_id) - ) - ) - if _skip_persist: + # message loads a transcript that reflects what was said; the assistant error text is + # a gateway-generated hint, not model output. Hidden-reasoning incomplete turns follow + # the same rule so peer-agent channels don't ingest them. + # Dedupe on platform message_id (Telegram retries after transient failures). + if event.message_id and await store.has_platform_message_id(sid, str(event.message_id)): logger.info( "Skipping duplicate user turn " "(message_id=%s) in session %s", - event.message_id, session_entry.session_id, + event.message_id, sid, ) else: - await self.async_session_store.append_to_transcript( - session_entry.session_id, - _user_entry, - skip_db=agent_persisted, - ) + await store.append_to_transcript(sid, _user_entry(), skip_db=agent_persisted) else: + # Only the NEW messages from this turn: use history_offset (what the agent saw), not + # len(history), which counts session_meta entries stripped before the agent saw them. history_len = agent_result.get("history_offset", len(history)) new_messages = agent_messages[history_len:] if len(agent_messages) > history_len else [] - - # If no new messages found (edge case), fall back to simple user/assistant if not new_messages: - _user_entry = { - "role": "user", - "content": ( - persist_user_message - if persist_user_message is not None - else message_text - ), - "timestamp": ( - persist_user_timestamp - if persist_user_timestamp is not None - else ts - ), - } - if persist_user_display_kind: - _user_entry["display_kind"] = persist_user_display_kind - if event.message_id: - _user_entry["message_id"] = str(event.message_id) - await self.async_session_store.append_to_transcript( - session_entry.session_id, - _user_entry, - skip_db=agent_persisted, - ) + # Edge case: fall back to simple user/assistant rows. + await store.append_to_transcript(sid, _user_entry(), skip_db=agent_persisted) if response: - await self.async_session_store.append_to_transcript( - session_entry.session_id, - {"role": "assistant", "content": response, "timestamp": ts}, + await store.append_to_transcript( + sid, {"role": "assistant", "content": response, "timestamp": ts}, skip_db=agent_persisted, ) else: - # Attach the inbound platform message_id to the first user entry written this - # turn so platform-level quote-resolution (e.g. Yuanbao QuoteContextMiddleware's + # Attach the inbound platform message_id to the first user entry written this turn + # so platform-level quote-resolution (e.g. Yuanbao QuoteContextMiddleware's # transcript fallback) can find earlier @bot messages by their original id. _user_msg_id_attached = False for msg in new_messages: - # Skip system messages (they're rebuilt each run) if msg.get("role") == "system": - continue - # Add timestamp to each message for debugging + continue # rebuilt each run entry = {**msg, "timestamp": ts} if ( not _user_msg_id_attached @@ -2558,27 +1767,21 @@ class GatewayTurnMixin: ): entry["message_id"] = str(event.message_id) _user_msg_id_attached = True - await self.async_session_store.append_to_transcript( - session_entry.session_id, entry, - skip_db=agent_persisted, - ) + await store.append_to_transcript(sid, entry, skip_db=agent_persisted) - # The agent persists token counts and model itself; keep only last_prompt_tokens here - # for context-window tracking and compression decisions. - await self.async_session_store.update_session( - session_entry.session_key, - last_prompt_tokens=agent_result.get("last_prompt_tokens", 0), + # The agent persists token counts and model itself; keep only last_prompt_tokens here for + # context-window tracking and compression decisions. + await store.update_session( + session_entry.session_key, last_prompt_tokens=agent_result.get("last_prompt_tokens", 0), touch_activity=not bool(getattr(event, "internal", False)), ) - # Re-baseline the cached agent's message_count snapshot now that ALL of this turn's - # transcript writes are done (flushed rows AND the first-turn `session_meta` marker). - # The cross-process coherence guard snapshots at agent-BUILD time and never refreshes - # on reuse, so our own writes would trigger a rebuild next turn (destroying prompt - # caching). MUST run after the session_meta append (that row bumps the count too). - await self._refresh_agent_cache_message_count( - session_key, session_entry.session_id - ) + # Re-baseline the cached agent's message_count snapshot now that ALL of this turn's writes + # are done (flushed rows AND the first-turn `session_meta` marker, which bumps the count + # too). The cross-process coherence guard snapshots at agent-BUILD time and never + # refreshes on reuse, so our own writes would trigger a rebuild next turn (destroying + # prompt caching). + await self._refresh_agent_cache_message_count(session_key, sid) async def _hmwa_deliver_turn_response( self, event, source, session_entry, session_key, run_generation, @@ -2587,126 +1790,94 @@ class GatewayTurnMixin: """Final delivery decisions: intentional silence, voice reply, streamed-turn media/footer. Returns the text for the adapter to send, or ``None`` when already delivered.""" # Intentional silence is a delivery decision, not a transcript mutation: the [SILENT] - # assistant turn stays persisted so later turns keep user/assistant alternation; only - # the outbound delivery is suppressed. + # assistant turn stays persisted so later turns keep user/assistant alternation. if _intentional_silence: - logger.info( - "Suppressing intentional silence marker for session %s", - session_entry.session_id, - ) + logger.info("Suppressing intentional silence marker for session %s", session_entry.session_id) response = "" - # Auto voice reply: send TTS audio before the text response - _already_sent = bool(agent_result.get("already_sent")) - # Skip when streaming TTS already delivered audio for this turn (#60671). - _stts_adapter = self._adapter_for_source(source) - _streaming_tts_done = ( - _stts_adapter is not None - and bool(getattr(_stts_adapter, "_streaming_tts_turn_completed", lambda *_a, **_k: False)(session_key, run_generation)) + adapter = self._adapter_for_source(source) + # Auto voice reply (TTS audio before the text) unless streaming TTS already delivered + # audio for this turn. + _streaming_tts_done = adapter is not None and bool( + getattr(adapter, "_streaming_tts_turn_completed", lambda *_a, **_k: False)(session_key, run_generation) ) - if ( - not _streaming_tts_done - and self._should_send_voice_reply(event, response, agent_messages, already_sent=_already_sent) + if not _streaming_tts_done and self._should_send_voice_reply( + event, response, agent_messages, already_sent=bool(agent_result.get("already_sent")), ): await self._send_voice_reply(event, response) - # Streamed responses still need MEDIA: files delivered before returning None (chunks - # carry the tags verbatim and post-processing is skipped when already_sent). Never skip - # when the agent failed: the error text is new content streaming didn't show. + # Streamed responses still need MEDIA: files delivered before returning None (chunks carry + # the tags verbatim and post-processing is skipped when already_sent). Never skip when the + # agent failed: the error text is new content streaming didn't show. if agent_result.get("already_sent") and not agent_result.get("failed"): - if response: - _media_adapter = self._adapter_for_source(source) - if _media_adapter: - await self._deliver_media_from_response( - response, event, _media_adapter, - ) - # Streaming already delivered the body text, but the footer was intentionally held - # back (see the `not already_sent` gate above). - if _footer_line: + if response and adapter: + await self._deliver_media_from_response(response, event, adapter) + # Streaming delivered the body, but the footer was held back (`not already_sent` gate). + if _footer_line and adapter: try: - _foot_adapter = self._adapter_for_source(source) - if _foot_adapter: - await _foot_adapter.send( - source.chat_id, - _footer_line, - metadata=self._thread_metadata_for_source(source, self._reply_anchor_for_event(event)), - ) + await adapter.send( + source.chat_id, _footer_line, + metadata=self._thread_metadata_for_source(source, self._reply_anchor_for_event(event)), + ) except Exception as _e: logger.debug("trailing footer send failed: %s", _e) - # This branch returns None so the adapter does not send the body twice. /loop and - # /goal hooks in _handle_message read the return value, so stash the delivered text - # on the event or those hooks never run and a /loop tick stays awaiting. + # Return None so the adapter does not send the body twice; /loop and /goal hooks in + # _handle_message read the return value, so stash the delivered text on the event or + # those hooks never run and a /loop tick stays awaiting. with suppress(Exception): event._streamed_final_response = str(response or "") return None return response + _STATUS_HINTS = { + 401: " Check your API key or run `claude /login` to refresh OAuth credentials.", + 402: " Your API balance or quota is exhausted. Check your provider dashboard.", + 529: " The API is temporarily overloaded. Please try again shortly.", + } + async def _hmwa_agent_error_reply( self, e, event, source, session_entry, session_key, history, message_text, persist_user_message, persist_user_timestamp, persist_user_display_kind, ): """``except Exception`` body of the agent turn: stop typing, log, persist the inbound user turn once, and build the sanitized user-facing error reply.""" - # Stop typing indicator on error too, retaining Slack thread/workspace - # routing so a failed turn cannot leave its status visible. + # Retain Slack thread/workspace routing so a failed turn cannot leave its status visible. await self._hmwa_stop_typing_for_turn(event, source) logger.exception("Agent error in session %s", session_key) # Crash-resilience for failures before AIAgent enters run_conversation() (e.g. provider/ # httpx client init): the agent can't persist the inbound turn there, so append the user - # message here once; if the agent already reached turn-start persistence the latest user - # row matches and we skip the duplicate. + # message here once; skip if the latest user row already matches it. try: if message_text is not None and session_entry is not None: - _already_persisted = False try: _recent_transcript = await self.async_session_store.load_transcript(session_entry.session_id) except Exception: _recent_transcript = [] + _expected_user_content = ( + persist_user_message if persist_user_message is not None else message_text + ) + _already_persisted = False for _msg in reversed(_recent_transcript[-10:]): if _msg.get("role") == "user": - _expected_user_content = ( - persist_user_message - if persist_user_message is not None - else message_text - ) - _already_persisted = (_msg.get("content") == _expected_user_content) + _already_persisted = _msg.get("content") == _expected_user_content break if not _already_persisted: - _user_entry = { - "role": "user", - "content": ( - persist_user_message - if persist_user_message is not None - else message_text - ), - "timestamp": ( - persist_user_timestamp - if persist_user_timestamp is not None - else time.time() - ), - } - if persist_user_display_kind: - _user_entry["display_kind"] = persist_user_display_kind - if getattr(event, "message_id", None): - _user_entry["message_id"] = str(event.message_id) await self.async_session_store.append_to_transcript( session_entry.session_id, - _user_entry, + self._hmwa_user_transcript_entry( + event, message_text, persist_user_message, persist_user_timestamp, + persist_user_display_kind, time.time(), + ), ) except Exception: logger.debug("Failed to persist inbound user message after agent exception", exc_info=True) - # Log full details server-side only; never expose raw exception - # types or messages to end users (info-leakage risk). - status_hint = "" + # Log full details server-side only; never expose raw exception types or messages to end + # users (info-leakage risk). status_code = getattr(e, "status_code", None) - _hist_len = len(history) - if status_code == 401: - status_hint = " Check your API key or run `claude /login` to refresh OAuth credentials." - elif status_code == 402: - status_hint = " Your API balance or quota is exhausted. Check your provider dashboard." - elif status_code == 429: - # Check if this is a plan usage limit (resets on a schedule) vs a transient rate limit + status_hint = self._STATUS_HINTS.get(status_code, "") + if status_code == 429: + # Plan usage limit (resets on a schedule) vs a transient rate limit _err_body = getattr(e, "response", None) _err_json = {} try: @@ -2726,12 +1897,10 @@ class GatewayTurnMixin: status_hint = " Your plan's usage limit has been reached. Please wait until it resets." else: status_hint = " You are being rate-limited. Please wait a moment and try again." - elif status_code == 529: - status_hint = " The API is temporarily overloaded. Please try again shortly." elif status_code in {400, 500}: # 400 on a large session is context overflow; 500 on a large session often means the # payload is too large for the API — treat it the same way. - if _hist_len > 50: + if len(history) > 50: return ( "⚠️ Session too large for the model's context window.\n" "Use /compact to compress the conversation, or " @@ -2744,91 +1913,83 @@ class GatewayTurnMixin: "Try again or use /reset to start a fresh session." ) + def _hmwa_discard_stale_result(self, source, _quick_key, run_generation): + """A newer run generation superseded this turn: drop its deferred post-delivery callback.""" + logger.info( + "Discarding stale agent result for %s — generation %d is no longer current", + _quick_key or "?", run_generation, + ) + _stale_adapter = self._adapter_for_source(source) + if getattr(type(_stale_adapter), "pop_post_delivery_callback", None) is not None: + _stale_adapter.pop_post_delivery_callback(_quick_key, generation=run_generation) + elif _stale_adapter and hasattr(_stale_adapter, "_post_delivery_callbacks"): + _stale_adapter._post_delivery_callbacks.pop(_quick_key, None) + async def _handle_message_with_agent(self, event, source, _quick_key: str, run_generation: int): """Inner handler that runs under the _running_agents sentinel guard.""" from gateway.run import _load_gateway_config _msg_start_time = time.time() _platform_name = source.platform.value if hasattr(source.platform, "value") else str(source.platform) - _msg_preview = (event.text or "")[:80].replace("\n", " ") - _reply_id = getattr(event, "reply_to_message_id", None) - _reply_txt = (getattr(event, "reply_to_text", None) or "")[:80].replace("\n", " ") logger.info( "inbound message: platform=%s user=%s chat=%s msg=%r reply_to_id=%s reply_to_text=%r", _platform_name, source.user_name or source.user_id or "unknown", - source.chat_id or "unknown", _msg_preview, _reply_id, _reply_txt, + source.chat_id or "unknown", (event.text or "")[:80].replace("\n", " "), + getattr(event, "reply_to_message_id", None), + (getattr(event, "reply_to_text", None) or "")[:80].replace("\n", " "), ) resolved = await self._hmwa_resolve_session(event, source) if resolved is None: return source, session_entry, session_key = resolved - _was_auto_reset, _is_new_session = await self._hmwa_open_session( - session_entry, session_key, source - ) + _was_auto_reset, _is_new_session = await self._hmwa_open_session(session_entry, session_key, source) - # Build session context context = build_session_context(source, self.config, session_entry) - - # Set session context variables for tools (task-local, concurrency-safe) + # Session context variables for tools (task-local, concurrency-safe) _session_env_tokens = self._set_session_env(context) - # Read privacy.redact_pii from config (re-read per message) - _redact_pii = False # Synthetic self-injected turns (batch completions, watch notifications, resume wake-ups) # arrive as MessageEvent(internal=True). Persist with display_kind="internal_notification" # so UIs render timeline notices, not user bubbles. display_kind is a DB-only sidecar # stripped from every provider-bound payload; role/content untouched. - persist_user_display_kind = ( - "internal_notification" if getattr(event, "internal", False) else None - ) + persist_user_display_kind = "internal_notification" if getattr(event, "internal", False) else None + _redact_pii = False # privacy.redact_pii, re-read per message try: - _pcfg = _load_gateway_config() - _redact_pii = bool((_pcfg.get("privacy") or {}).get("redact_pii", False)) + _redact_pii = bool((_load_gateway_config().get("privacy") or {}).get("redact_pii", False)) except Exception: pass - # Build the context prompt. The render is pinned per session, keyed by a hash of the exact - # renderer inputs (_ephemeral_change_key): a hit reuses the pinned bytes so the system prompt - # cannot drift turn-over-turn; a miss (thread rename, /sethome, redact_pii flip) re-renders. - context_prompt = self._pinned_session_context_prompt( - context, _redact_pii, session_key - ) + # The context prompt render is pinned per session, keyed by a hash of the exact renderer + # inputs (_ephemeral_change_key): a hit reuses the pinned bytes so the system prompt cannot + # drift turn-over-turn; a miss (thread rename, /sethome, redact_pii flip) re-renders. + context_prompt = self._pinned_session_context_prompt(context, _redact_pii, session_key) # Per-turn must-deliver notes ride the user message via the api_content sidecar (staged # below, consumed in run_sync → build_turn_context), NOT context_prompt: appending them to # the ephemeral system prompt guaranteed a turn1→turn2 diff and a full agent rebuild. turn_sidecar_notes: List[str] = [] - - # If the previous session expired and was auto-reset, deliver a notice - # so the agent knows this is a fresh conversation (not an intentional /reset). if _was_auto_reset: await self._hmwa_deliver_auto_reset_notice(session_entry, source, turn_sidecar_notes) - # Auto-load skill(s) for topic/channel bindings (Telegram DM Topics, Discord - # channel_skill_bindings). Supports a single name or ordered list. Only inject on NEW - # sessions — ongoing conversations already carry the skill content in their history. + # Auto-load skill(s) for topic/channel bindings (single name or ordered list) — only on NEW + # sessions; ongoing conversations already carry the skill content in their history. _auto = getattr(event, "auto_skill", None) if _is_new_session and _auto: self._hmwa_auto_load_skills(event, _auto, _quick_key, session_key) - # Turn lease: session resolution is FINAL here (see _hmwa_acquire_turn_lease). - await self._hmwa_acquire_turn_lease( - _quick_key, run_generation, session_entry, _session_env_tokens - ) + await self._hmwa_acquire_turn_lease(_quick_key, run_generation, session_entry, _session_env_tokens) # A turn only becomes durable recovery work after it owns (or has explicitly degraded past) # the per-session lease. Marking before the await above would falsely recover an alias- # routed message that never began processing if the gateway died while it was still waiting. await self._mark_durable_active_turn(event, session_entry.session_key) - # Load conversation history from transcript. An unreadable canonical store is not an empty - # conversation: stop before the agent can invent continuity from a plausible-looking []. - # This return happens before the broad cleanup finally below, so restore task-local context - # here; the outer dispatch still clears the durable marker and turn lease. + # An unreadable canonical store is not an empty conversation: stop before the agent can + # invent continuity from a plausible-looking []. This return happens before the broad + # cleanup finally below, so restore task-local context here; the outer dispatch still + # clears the durable marker and turn lease. try: - history = await self.async_session_store.load_transcript( - session_entry.session_id - ) + history = await self.async_session_store.load_transcript(session_entry.session_id) except TranscriptReadError: self._clear_session_env(_session_env_tokens) return ( @@ -2838,35 +1999,27 @@ class GatewayTurnMixin: "if you intentionally want to start a new conversation." ) - # Session hygiene: auto-compress pathologically large transcripts before the agent starts. history = await self._hmwa_run_session_hygiene( event, source, session_entry, session_key, history, _quick_key, run_generation, ) await self._hmwa_first_contact_notes(source, history, turn_sidecar_notes) - # Voice channel awareness: deliver voice channel state (who is present / speaking) on the - # user message, ONLY when changed since the previous turn. It differs almost every turn, and - # in the ephemeral system prompt it forced a full agent rebuild + prompt-cache re-key per - # message; the system prompt carries a static pointer line instead (gateway/session.py). + # Voice channel state (who is present / speaking) rides the user message ONLY when changed + # since the previous turn; in the ephemeral system prompt it forced a full agent rebuild + + # prompt-cache re-key per message (the prompt carries a static pointer line instead). _vc_note = self._voice_channel_sidecar_note(event, source, session_key) if _vc_note: turn_sidecar_notes.append(_vc_note) - # Auto-analyze user images: run the vision tool eagerly so the model always gets a text - # description plus the local path for re-examination via vision_analyze. Filter to image - # media_type so documents/audio in the same message are not sent to the vision tool. + # Auto-analyze user images (vision tool eagerly, image media_type only) so the model always + # gets a text description plus the local path for re-examination via vision_analyze. message_text = await self._prepare_profile_scoped_inbound_message_text( - event=event, - source=source, - history=history, - session_key=session_key, + event=event, source=source, history=history, session_key=session_key, ) if message_text is None: return - # Capture the platform event time as message metadata and keep the persisted transcript - # clean; only the in-context render is gated behind gateway.message_timestamps.enabled. message_text, persist_user_message, persist_user_timestamp = ( self._hmwa_apply_message_timestamp(event, message_text) ) @@ -2878,14 +2031,9 @@ class GatewayTurnMixin: # Bind this run generation to the adapter's active-session event so deferred post-delivery # callbacks can be released by the same run that registered them. - self._bind_adapter_run_generation( - self._adapter_for_source(source), - session_key, - run_generation, - ) + self._bind_adapter_run_generation(self._adapter_for_source(source), session_key, run_generation) try: - # Emit agent:start hook hook_ctx = { "platform": source.platform.value if source.platform else "", "user_id": source.user_id, @@ -2897,25 +2045,17 @@ class GatewayTurnMixin: } await self.hooks.emit("agent:start", hook_ctx) - # Run the agent. Capture the session id that this run was launched against so post-run - # compression publication can be identity-guarded below; a /new or another lifecycle - # transition may move session_entry.session_id while the old run is still unwinding. + # Capture the session id this run launches against so post-run compression publication + # can be identity-guarded; a /new or another lifecycle transition may move + # session_entry.session_id while the old run is still unwinding. _run_start_session_id = session_entry.session_id _turn_started_monotonic = time.monotonic() agent_result = await self._run_agent( - message=message_text, - context_prompt=context_prompt, - history=history, - source=source, - session_id=_run_start_session_id, - session_key=session_key, - run_generation=run_generation, - event_message_id=self._reply_anchor_for_event(event), - inbound_message_id=( - str(event.message_id) if event.message_id else None - ), - channel_prompt=event.channel_prompt, - moa_config=getattr(event, "_moa_config", None), + message=message_text, context_prompt=context_prompt, history=history, source=source, + session_id=_run_start_session_id, session_key=session_key, + run_generation=run_generation, event_message_id=self._reply_anchor_for_event(event), + inbound_message_id=str(event.message_id) if event.message_id else None, + channel_prompt=event.channel_prompt, moa_config=getattr(event, "_moa_config", None), persist_user_message=persist_user_message, persist_user_timestamp=persist_user_timestamp, persist_user_display_kind=persist_user_display_kind, @@ -2926,19 +2066,7 @@ class GatewayTurnMixin: await self._hmwa_stop_typing_for_turn(event, source) if not self._is_session_run_current(_quick_key, run_generation): - logger.info( - "Discarding stale agent result for %s — generation %d is no longer current", - _quick_key or "?", - run_generation, - ) - _stale_adapter = self._adapter_for_source(source) - if getattr(type(_stale_adapter), "pop_post_delivery_callback", None) is not None: - _stale_adapter.pop_post_delivery_callback( - _quick_key, - generation=run_generation, - ) - elif _stale_adapter and hasattr(_stale_adapter, "_post_delivery_callbacks"): - _stale_adapter._post_delivery_callbacks.pop(_quick_key, None) + self._hmwa_discard_stale_result(source, _quick_key, run_generation) return None response, _intentional_silence, agent_messages = await self._hmwa_shape_agent_response( @@ -2947,6 +2075,7 @@ class GatewayTurnMixin: ) response = self._hmwa_prepend_reasoning(agent_result, response, source, _intentional_silence) _footer_line = self._hmwa_runtime_footer_line(agent_result, source, _turn_seconds) + # Streaming already delivered the body: the footer goes out as a trailing send instead. if _footer_line and response and not agent_result.get("already_sent") and not _intentional_silence: response = f"{response}\n\n{_footer_line}" await self._hmwa_post_turn_hooks(hook_ctx, agent_result, response) @@ -2958,15 +2087,9 @@ class GatewayTurnMixin: agent_result, response, session_entry, session_key, source, ) await self._hmwa_persist_turn_transcript( - event=event, - source=source, - session_entry=session_entry, - session_key=session_key, - agent_result=agent_result, - agent_messages=agent_messages, - history=history, - response=response, - message_text=message_text, + event=event, source=source, session_entry=session_entry, session_key=session_key, + agent_result=agent_result, agent_messages=agent_messages, history=history, + response=response, message_text=message_text, persist_user_message=persist_user_message, persist_user_timestamp=persist_user_timestamp, persist_user_display_kind=persist_user_display_kind, @@ -2988,154 +2111,106 @@ class GatewayTurnMixin: # Restore session context variables to their pre-handler state self._clear_session_env(_session_env_tokens) - def _reset_notice_session_info(self, source: SessionSource) -> str: - """Session-info block for the auto-reset notice, profile-scoped. + def _profile_scope_for_source(self, source: SessionSource): + """``_profile_runtime_scope`` for ``source``'s profile when multiplexing, else a no-op context. - Under multiplexing, resolve model/provider/context inside the profile serving ``source`` - (mirrors ``_run_agent``'s gating) or the banner advertises the base config's model. Call - via ``asyncio.to_thread``: resolution can block (credential refresh, context-length - probes), and the scope is entered here so contextvars behave in the worker thread. + Under multiplexing config/skills/memory resolve to the source profile's home AND credentials + come from its secret scope (never process-global ``os.environ``). """ from gateway.run import _profile_runtime_scope if getattr(getattr(self, "config", None), "multiplex_profiles", False): - with _profile_runtime_scope(self._resolve_profile_home_for_source(source)): - return self._format_session_info() - return self._format_session_info() + return _profile_runtime_scope(self._resolve_profile_home_for_source(source)) + return nullcontext() + + def _reset_notice_session_info(self, source: SessionSource) -> str: + """Session-info block for the auto-reset notice, resolved inside the profile serving ``source``. + + Call via ``asyncio.to_thread``: resolution can block (credential refresh, context-length + probes), and the scope is entered here so contextvars behave in the worker thread. + """ + with self._profile_scope_for_source(source): + return self._format_session_info() def _format_session_info(self) -> str: - """Resolve current model config and return a formatted info block. - - Surfaces model, provider, context length, and endpoint so gateway users can immediately - see if context detection went wrong (e.g. local models falling to the 128K default). - """ + """Model / provider / context-length / endpoint block so users can spot bad context detection.""" from gateway.run import _resolve_gateway_model_context resolved = _resolve_gateway_model_context() - model = resolved.model - provider = resolved.provider - base_url = resolved.base_url context_length = resolved.context_length - - # Format context source hint - if resolved.context_source == "config": - ctx_source = "config" - elif resolved.context_source == "default": - ctx_source = "default — set model.context_length in config to override" - else: - ctx_source = "detected" - - # Format context length for display + ctx_source = { + "config": "config", + "default": "default — set model.context_length in config to override", + }.get(resolved.context_source, "detected") if context_length >= 1_000_000: ctx_display = f"{context_length / 1_000_000:.1f}M" elif context_length >= 1_000: ctx_display = f"{context_length // 1_000}K" else: ctx_display = str(context_length) - lines = [ - f"◆ Model: `{model}`", - f"◆ Provider: {provider or 'openrouter'}", + f"◆ Model: `{resolved.model}`", + f"◆ Provider: {resolved.provider or 'openrouter'}", f"◆ Context: {ctx_display} tokens ({ctx_source})", ] - - # Show endpoint for local/custom setups + base_url = resolved.base_url if base_url and base_url_hostname(base_url) in ("localhost", "127.0.0.1", "0.0.0.0"): lines.append(f"◆ Endpoint: {base_url}") - return "\n".join(lines) async def _run_background_task( - self, - prompt: str, - source: "SessionSource", - task_id: str, - event_message_id: Optional[str] = None, - media_urls: Optional[List[str]] = None, + self, prompt: str, source: "SessionSource", task_id: str, + event_message_id: Optional[str] = None, media_urls: Optional[List[str]] = None, media_types: Optional[List[str]] = None, ) -> None: - """Profile-scoping wrapper around the background agent task. - - When multiplexing is active, resolve the inbound source's profile and run the whole task - inside ``_profile_runtime_scope`` so credentials resolve from that profile's secret - scope. Mirrors the pattern in ``_run_agent``. - """ - from gateway.run import _profile_runtime_scope - if not getattr(getattr(self, "config", None), "multiplex_profiles", False): - return await self._run_background_task_inner( - prompt, source, task_id, event_message_id, media_urls, media_types, - ) - - profile_home = self._resolve_profile_home_for_source(source) - with _profile_runtime_scope(profile_home): + """Profile-scoping wrapper around the background agent task (mirrors ``_run_agent``).""" + with self._profile_scope_for_source(source): return await self._run_background_task_inner( prompt, source, task_id, event_message_id, media_urls, media_types, ) def _resolve_enabled_toolsets_for_source( - self, - user_config: dict, - source: "SessionSource", - platform_key: str, + self, user_config: dict, source: "SessionSource", platform_key: str, ) -> list: - """Resolve enabled toolsets for an agent run, honoring per-source overrides. + """Enabled toolsets for an agent run, honoring an adapter ``toolsets_for_source()`` override. - An adapter ``toolsets_for_source()`` override (e.g. per-route webhook toolsets) is - validated through the SAME ``_get_platform_tools`` path as normal platform config, so - unknown and platform-restricted toolsets are dropped rather than trusted. Absent an - override, falls back to ``platform_toolsets.``. + The override (e.g. per-route webhook toolsets) is validated through the SAME + ``_get_platform_tools`` path as normal platform config, so unknown and platform-restricted + toolsets are dropped rather than trusted. Absent an override: ``platform_toolsets.``. """ from hermes_cli.tools_config import _get_platform_tools - - override = None try: adapter = self._adapter_for_source(source) - if adapter is not None: - override = adapter.toolsets_for_source(source) + override = adapter.toolsets_for_source(source) if adapter is not None else None except Exception: override = None - if override and isinstance(override, list): - cfg = dict(user_config) - pts = dict(cfg.get("platform_toolsets") or {}) - pts[platform_key] = [str(t) for t in override] - cfg["platform_toolsets"] = pts - return sorted(_get_platform_tools(cfg, platform_key)) - + pts = dict(user_config.get("platform_toolsets") or {}) + pts[platform_key] = [str(x) for x in override] + user_config = {**user_config, "platform_toolsets": pts} return sorted(_get_platform_tools(user_config, platform_key)) async def _run_background_task_inner( - self, - prompt: str, - source: "SessionSource", - task_id: str, - event_message_id: Optional[str] = None, - media_urls: Optional[List[str]] = None, + self, prompt: str, source: "SessionSource", task_id: str, + event_message_id: Optional[str] = None, media_urls: Optional[List[str]] = None, media_types: Optional[List[str]] = None, ) -> None: """Execute a background agent task and deliver the result to the chat.""" from gateway.run import ( - _checkpoint_agent_kwargs, - _current_max_iterations, - _load_gateway_config, + _checkpoint_agent_kwargs, _current_max_iterations, _load_gateway_config, _platform_config_key, ) from run_agent import AIAgent media_urls = media_urls or [] media_types = media_types or [] - adapter = self._adapter_for_source(source) if not adapter: logger.warning("No adapter for platform %s in background task %s", source.platform, task_id) return - _thread_metadata = self._thread_metadata_for_source(source, event_message_id) try: user_config = _load_gateway_config() - model, runtime_kwargs = self._resolve_session_agent_runtime( - source=source, - user_config=user_config, - ) + model, runtime_kwargs = self._resolve_session_agent_runtime(source=source, user_config=user_config) if not runtime_kwargs.get("api_key"): await adapter.send( source.chat_id, @@ -3145,40 +2220,29 @@ class GatewayTurnMixin: return platform_key = _platform_config_key(source.platform) - - enabled_toolsets = self._resolve_enabled_toolsets_for_source( - user_config, source, platform_key - ) - agent_cfg = user_config.get("agent") or {} + enabled_toolsets = self._resolve_enabled_toolsets_for_source(user_config, source, platform_key) from agent.skill_utils import parse_config_string_list - - disabled_toolsets = parse_config_string_list(agent_cfg.get("disabled_toolsets")) or None - + disabled_toolsets = parse_config_string_list( + (user_config.get("agent") or {}).get("disabled_toolsets") + ) or None pr = self._provider_routing max_iterations = _current_max_iterations() - reasoning_config = self._resolve_session_reasoning_config( - source=source, model=model - ) + reasoning_config = self._resolve_session_reasoning_config(source=source, model=model) self._reasoning_config = reasoning_config self._service_tier = self._resolve_session_service_tier(source=source) turn_route = self._resolve_turn_agent_config(prompt, model, runtime_kwargs) - # Enrich the prompt with image descriptions so the background - # agent can see user-attached images (same as the main flow). + # Enrich the prompt with image descriptions (same as the main flow). enriched_prompt = prompt - if media_urls: - image_paths = [] - for i, path in enumerate(media_urls): - mtype = media_types[i] if i < len(media_types) else "" - if mtype.startswith("image/"): - image_paths.append(path) - if image_paths: - try: - enriched_prompt = await self._enrich_message_with_vision( - prompt, image_paths, - ) - except Exception as e: - logger.warning("Background task vision enrichment failed: %s", e) + image_paths = [ + path for i, path in enumerate(media_urls) + if (media_types[i] if i < len(media_types) else "").startswith("image/") + ] + if image_paths: + try: + enriched_prompt = await self._enrich_message_with_vision(prompt, image_paths) + except Exception as e: + logger.warning("Background task vision enrichment failed: %s", e) def run_sync(): agent = AIAgent( @@ -3209,14 +2273,11 @@ class GatewayTurnMixin: chat_type=source.chat_type, thread_id=source.thread_id, session_db=getattr(self._session_db, "_db", self._session_db), - # Reload from disk — do not reuse the startup snapshot (#60955). + # Reload from disk — do not reuse the startup snapshot. fallback_model=self._refresh_fallback_model(), ) try: - return agent.run_conversation( - user_message=enriched_prompt, - task_id=task_id, - ) + return agent.run_conversation(user_message=enriched_prompt, task_id=task_id) finally: self._cleanup_agent_resources(agent) @@ -3225,109 +2286,66 @@ class GatewayTurnMixin: response = result.get("final_response", "") if result else "" if not response and result and result.get("error"): response = f"Error: {result['error']}" - - # Background tasks start a fresh conversation, so history_offset=0: every message in the - # run belongs to this turn. Mirrors the repair on the main turn path. + # Fresh conversation, so history_offset=0: every message in the run belongs to this turn. if response: - response = repair_explicit_computer_use_media_paths( - response, - result.get("messages", []), - ) + response = repair_explicit_computer_use_media_paths(response, result.get("messages", [])) - # Extract media files from the response + preview = prompt[:60] + ("..." if len(prompt) > 60 else "") + header = f'✅ Background task complete\nPrompt: "{preview}"\n\n' + images, media_files, text_content = [], [], "" if response: media_files, response = adapter.extract_media(response) - from gateway.platforms.base import BasePlatformAdapter media_files = BasePlatformAdapter.filter_media_delivery_paths(media_files) images, text_content = adapter.extract_images(response) - - preview = prompt[:60] + ("..." if len(prompt) > 60 else "") - header = f'✅ Background task complete\nPrompt: "{preview}"\n\n' - - if text_content: - await adapter.send( - chat_id=source.chat_id, - content=header + text_content, - metadata=_thread_metadata, - ) - elif not images and not media_files: - await adapter.send( - chat_id=source.chat_id, - content=header + "(No response generated)", - metadata=_thread_metadata, - ) - - # Send extracted images - for image_url, alt_text in (images or []): - with suppress(Exception): - await adapter.send_image( - chat_id=source.chat_id, - image_url=image_url, - caption=alt_text, - metadata=_thread_metadata, - ) - - # Route each media file by type so a TTS clip arrives as a voice bubble and a clip - # as a video rather than a generic document. Mirrors the streaming + kanban paths. - from gateway.platforms.base import ( - should_send_media_as_audio as _should_send_media_as_audio, - ) - _IMAGE_EXTS = {".png", ".jpg", ".jpeg", ".gif", ".webp"} - _VIDEO_EXTS = {".mp4", ".mov", ".avi", ".mkv", ".webm", ".3gp"} - for media_path, _is_voice in (media_files or []): - _ext = os.path.splitext(media_path)[1].lower() - try: - if _should_send_media_as_audio(source.platform, _ext, _is_voice): - await adapter.send_voice( - chat_id=source.chat_id, - audio_path=media_path, - metadata=_thread_metadata, - is_voice=_is_voice, - ) - elif _ext in _VIDEO_EXTS: - await adapter.send_video( - chat_id=source.chat_id, - video_path=media_path, - metadata=_thread_metadata, - ) - elif _ext in _IMAGE_EXTS: - await adapter.send_image_file( - chat_id=source.chat_id, - image_path=media_path, - metadata=_thread_metadata, - ) - else: - await adapter.send_document( - chat_id=source.chat_id, - file_path=media_path, - metadata=_thread_metadata, - ) - except Exception: - pass - else: - preview = prompt[:60] + ("..." if len(prompt) > 60 else "") + if text_content: + await adapter.send(chat_id=source.chat_id, content=header + text_content, metadata=_thread_metadata) + elif not images and not media_files: await adapter.send( - chat_id=source.chat_id, - content=f'✅ Background task complete\nPrompt: "{preview}"\n\n(No response generated)', - metadata=_thread_metadata, + chat_id=source.chat_id, content=header + "(No response generated)", metadata=_thread_metadata, ) + for image_url, alt_text in (images or []): + with suppress(Exception): + await adapter.send_image( + chat_id=source.chat_id, image_url=image_url, caption=alt_text, metadata=_thread_metadata, + ) + # Route each media file by type so a TTS clip arrives as a voice bubble and a clip as a + # video rather than a generic document. Mirrors the streaming + kanban paths. + from gateway.platforms.base import should_send_media_as_audio as _should_send_media_as_audio + from gateway.run_notifications import _IMAGE_EXTS, _VIDEO_EXTS + for media_path, _is_voice in (media_files or []): + _ext = os.path.splitext(media_path)[1].lower() + with suppress(Exception): + if _should_send_media_as_audio(source.platform, _ext, _is_voice): + await adapter.send_voice( + chat_id=source.chat_id, audio_path=media_path, metadata=_thread_metadata, + is_voice=_is_voice, + ) + elif _ext in _VIDEO_EXTS: + await adapter.send_video( + chat_id=source.chat_id, video_path=media_path, metadata=_thread_metadata, + ) + elif _ext in _IMAGE_EXTS: + await adapter.send_image_file( + chat_id=source.chat_id, image_path=media_path, metadata=_thread_metadata, + ) + else: + await adapter.send_document( + chat_id=source.chat_id, file_path=media_path, metadata=_thread_metadata, + ) except Exception as e: logger.exception("Background task %s failed", task_id) with suppress(Exception): await adapter.send( - chat_id=source.chat_id, - content=f"❌ Background task {task_id} failed: {e}", + chat_id=source.chat_id, content=f"❌ Background task {task_id} failed: {e}", metadata=_thread_metadata, ) async def _execute_mcp_reload(self, event: MessageEvent) -> str: - """Actually disconnect, reconnect, and notify MCP tool changes. + """Disconnect, reconnect, and notify MCP tool changes (shared by button / text / no-confirm paths). - Split out so the confirmation wrapper can invoke the same path for button, text reply, - or disabled confirm gate. Under multiplex the reload runs inside the requesting profile's - runtime scope (entered here when the caller did not) and only that profile's servers are - torn down and rediscovered. + Under multiplex the reload runs inside the requesting profile's runtime scope (entered here + when the caller did not) and only that profile's servers are torn down and rediscovered. """ from gateway.run import _profile_runtime_scope multiplex = bool(getattr(self.config, "multiplex_profiles", False)) @@ -3349,50 +2367,36 @@ class GatewayTurnMixin: if reload_scope is None or _server_scope_keys.get(name) == reload_scope } - # Capture old server names before shutdown old_servers = _scoped_server_names() - - # Read new config before shutting down, so we know what will be added/removed - # Shutdown existing connections - await self._run_in_executor_with_context( - lambda: shutdown_mcp_servers(scope=reload_scope) - ) + await self._run_in_executor_with_context(lambda: shutdown_mcp_servers(scope=reload_scope)) # Explicit reload also re-probes tool availability (check_fn). reprobe_tool_availability() - - # Reconnect by discovering tools (reads config.yaml fresh) + # Reconnect by discovering tools (reads config.yaml fresh). new_tools = await self._run_in_executor_with_context(discover_mcp_tools) - # Compute what changed connected_servers = _scoped_server_names() if reload_scope is not None: from tools.mcp_tool import _mcp_tool_server_names - with _lock: - new_tools = [ - n for n in new_tools - if _mcp_tool_server_names.get(n) in connected_servers - ] - + new_tools = [n for n in new_tools if _mcp_tool_server_names.get(n) in connected_servers] added = connected_servers - old_servers removed = old_servers - connected_servers reconnected = connected_servers & old_servers lines = [t("gateway.reload_mcp.header")] - if reconnected: - lines.append(t("gateway.reload_mcp.reconnected", names=", ".join(sorted(reconnected)))) - if added: - lines.append(t("gateway.reload_mcp.added", names=", ".join(sorted(added)))) - if removed: - lines.append(t("gateway.reload_mcp.removed", names=", ".join(sorted(removed)))) + for names, key in ( + (reconnected, "gateway.reload_mcp.reconnected"), + (added, "gateway.reload_mcp.added"), (removed, "gateway.reload_mcp.removed"), + ): + if names: + lines.append(t(key, names=", ".join(sorted(names)))) if not connected_servers: lines.append(t("gateway.reload_mcp.none_connected")) else: lines.append(t("gateway.reload_mcp.tools_available", tools=len(new_tools), servers=len(connected_servers))) - # Refresh cached agents so existing sessions see new MCP tools on their next turn — - # without this, the user has to `/new` (which discards conversation history) to pick up - # tools from a server that was just added or reconnected. + # Refresh cached agents so existing sessions see new MCP tools on their next turn without + # a history-destroying `/new`. try: from tools.mcp_tool import refresh_agent_mcp_tools _cache = getattr(self, "_agent_cache", None) @@ -3400,10 +2404,7 @@ class GatewayTurnMixin: if _cache_lock is not None and _cache: # Multiplex: only this profile's sessions; rebuilding another profile's agent in # this scope would hand it this profile's tool registry. - _ns_prefix = ( - _session_key_namespace(event.source.profile) + ":" - if multiplex else None - ) + _ns_prefix = _session_key_namespace(event.source.profile) + ":" if multiplex else None with _cache_lock: for _sess_key, _entry in list(_cache.items()): if _ns_prefix and not str(_sess_key).startswith(_ns_prefix): @@ -3415,39 +2416,28 @@ class GatewayTurnMixin: if _agent is None: continue # Preserve each cached agent's build-time toolset selection EXACTLY: a - # gateway session built with a restricted enabled_toolsets (e.g. - # ["safe"]) must NOT silently gain tools after a reload. Unlike the - # CLI/TUI /reload-mcp (one user re-applying their own config), gateway - # agents are per-session and may be deliberately locked down. + # session built with restricted enabled_toolsets (e.g. ["safe"]) must NOT + # silently gain tools after a reload — gateway agents may be locked down. refresh_agent_mcp_tools(_agent, quiet_mode=True) except Exception as _exc: - logger.debug( - "Failed to update cached agent tools after MCP reload: %s", - _exc, - ) + logger.debug("Failed to update cached agent tools after MCP reload: %s", _exc) # Inject a message at the END of the session history so the model knows tools changed # next turn; appending after all existing messages preserves the prompt-cache prefix. - change_parts = [] - if added: - change_parts.append(f"Added servers: {', '.join(sorted(added))}") - if removed: - change_parts.append(f"Removed servers: {', '.join(sorted(removed))}") - if reconnected: - change_parts.append(f"Reconnected servers: {', '.join(sorted(reconnected))}") + change_parts = [ + f"{label} servers: {', '.join(sorted(names))}" + for label, names in (("Added", added), ("Removed", removed), ("Reconnected", reconnected)) + if names + ] tool_summary = f"{len(new_tools)} MCP tool(s) now available" if new_tools else "No MCP tools available" change_detail = ". ".join(change_parts) + ". " if change_parts else "" reload_msg = { "role": "user", "content": f"[IMPORTANT: MCP servers have been reloaded. {change_detail}{tool_summary}. The tool list for this conversation has been updated accordingly.]", } - try: + with suppress(Exception): # Best-effort; don't fail the reload over a transcript write session_entry = await self.async_session_store.get_or_create_session(event.source) - await self.async_session_store.append_to_transcript( - session_entry.session_id, reload_msg - ) - except Exception: - pass # Best-effort; don't fail the reload over a transcript write + await self.async_session_store.append_to_transcript(session_entry.session_id, reload_msg) return "\n".join(lines) @@ -3456,129 +2446,93 @@ class GatewayTurnMixin: return t("gateway.reload_mcp.failed", error=e) def _get_proxy_url(self) -> Optional[str]: - """Return the proxy URL if proxy mode is configured, else None. - - GATEWAY_PROXY_URL env var (Docker-friendly) wins over ``gateway.proxy_url`` in config.yaml. - """ + """Proxy URL if proxy mode is configured (GATEWAY_PROXY_URL env wins over ``gateway.proxy_url``).""" from gateway.run import _load_gateway_config url = os.getenv("GATEWAY_PROXY_URL", "").strip() - if url: - return url.rstrip("/") - cfg = _load_gateway_config() - url = (cfg.get("gateway") or {}).get("proxy_url") - url = (url or "").strip() - if url: - return url.rstrip("/") - return None + if not url: + url = ((_load_gateway_config().get("gateway") or {}).get("proxy_url") or "").strip() + return url.rstrip("/") if url else None def _build_stream_consumer_config( - self, - source: "SessionSource", - scfg: Any, - adapter: Any, - *, - on_missing_cursor: str, + self, source: "SessionSource", scfg: Any, adapter: Any, *, on_missing_cursor: str, ) -> "tuple[Any, Optional[Callable[[], None]]]": """Build the shared ``StreamConsumerConfig`` and optional Telegram pause-typing closure. ``on_missing_cursor`` handles adapters with ``SUPPORTS_MESSAGE_EDITING = False``: ``"fallback"`` (proxy path) streams with an empty cursor; ``"raise"`` (in-process path) - raises ``RuntimeError`` so the caller's ``except`` skips streaming entirely. Returns - ``(consumer_cfg, pause_typing_before_finalize)``. + raises ``RuntimeError`` so the caller's ``except`` skips streaming entirely. """ from gateway.stream_consumer import StreamConsumerConfig _pause_typing_before_finalize = None if source.platform == Platform.TELEGRAM and hasattr(adapter, "pause_typing_for_chat"): - def _pause_typing_before_finalize( - _adapter=adapter, - _chat_id=source.chat_id, - ) -> None: + def _pause_typing_before_finalize(_adapter=adapter, _chat_id=source.chat_id) -> None: _adapter.pause_typing_for_chat(_chat_id) # Platforms that can't edit sent messages (e.g. QQ, WeChat) skip streaming entirely: the - # partial first message could never be updated, yielding duplicates (partial + final). + # partial first message could never be updated, yielding duplicates (partial + final) — + # unless they have a native-streaming transport (e.g. WeCom msgtype "stream"), which the + # consumer's native branch delivers whole. _adapter_supports_edit = getattr(adapter, "SUPPORTS_MESSAGE_EDITING", True) - # Adapters that can't edit but have a native-streaming transport (e.g. WeCom msgtype "stream" - # via send_stream_frame) pass the gate — the consumer's native branch delivers the full turn. - _adapter_supports_native_stream = bool(getattr( - adapter, "SUPPORTS_NATIVE_STREAMING", False, - )) - if ( - not _adapter_supports_edit - and not _adapter_supports_native_stream - and on_missing_cursor == "raise" - ): + _adapter_supports_native_stream = bool(getattr(adapter, "SUPPORTS_NATIVE_STREAMING", False)) + if not _adapter_supports_edit and not _adapter_supports_native_stream and on_missing_cursor == "raise": raise RuntimeError("skip streaming for non-editable platform") _effective_cursor = scfg.cursor if _adapter_supports_edit else "" - # Some Matrix clients render the streaming cursor as a visible tofu/white-box artifact: keep - # streaming text on Matrix, but suppress the cursor. - _buffer_only = False - if source.platform == Platform.MATRIX: + # Some Matrix clients render the cursor as a tofu/white-box artifact: stream text, no cursor. + _buffer_only = source.platform == Platform.MATRIX + if _buffer_only: _effective_cursor = "" - _buffer_only = True # Fresh-final applies to Telegram only — other platforms edit in place cheaply (Discord, Slack) # or lack the edit-timestamp-stays-stale problem. _fresh_final_secs = ( float(getattr(scfg, "fresh_final_after_seconds", 0.0) or 0.0) - if source.platform == Platform.TELEGRAM - else 0.0 + if source.platform == Platform.TELEGRAM else 0.0 ) _consumer_cfg = StreamConsumerConfig( - edit_interval=scfg.edit_interval, - buffer_threshold=scfg.buffer_threshold, - cursor=_effective_cursor, - buffer_only=_buffer_only, - fresh_final_after_seconds=_fresh_final_secs, - transport=scfg.transport or "edit", + edit_interval=scfg.edit_interval, buffer_threshold=scfg.buffer_threshold, + cursor=_effective_cursor, buffer_only=_buffer_only, + fresh_final_after_seconds=_fresh_final_secs, transport=scfg.transport or "edit", chat_type=getattr(source, "chat_type", "") or "", ) return _consumer_cfg, _pause_typing_before_finalize + def _run_still_current_fn(self, session_key: Optional[str], run_generation: Optional[int]) -> Callable[[], bool]: + """Predicate: does this run's generation still own ``session_key``? (always True when untracked).""" + def _run_still_current() -> bool: + if run_generation is None or not session_key: + return True + return self._is_session_run_current(session_key, run_generation) + return _run_still_current + + @staticmethod + def _proxy_error_result(text: str) -> Dict[str, Any]: + return {"final_response": text, "messages": [], "api_calls": 0, "tools": []} + async def _run_agent_via_proxy( - self, - message: str, - context_prompt: str, - history: List[Dict[str, Any]], - source: "SessionSource", - session_id: str, - session_key: str = None, - run_generation: Optional[int] = None, - event_message_id: Optional[str] = None, + self, message: str, context_prompt: str, history: List[Dict[str, Any]], + source: "SessionSource", session_id: str, session_key: str = None, + run_generation: Optional[int] = None, event_message_id: Optional[str] = None, ) -> Dict[str, Any]: """Forward the message to a remote Hermes API server instead of running a local AIAgent. - This lets a Docker container handle Matrix E2EE while the actual agent runs on the host - with full access to local files, memory, skills, and a unified session store. + Lets a Docker container handle Matrix E2EE while the agent runs on the host with full + access to local files, memory, skills, and a unified session store. """ from gateway.run import ( - _GATEWAY_PROXY_SSE_BUFFER_MAX_CHARS, - _load_gateway_config, - _platform_config_key, + _GATEWAY_PROXY_SSE_BUFFER_MAX_CHARS, _load_gateway_config, _platform_config_key ) try: from aiohttp import ClientSession as _AioClientSession, ClientTimeout except ImportError: - return { - "final_response": "⚠️ Proxy mode requires aiohttp. Install with: pip install aiohttp", - "messages": [], - "api_calls": 0, - "tools": [], - } + return self._proxy_error_result("⚠️ Proxy mode requires aiohttp. Install with: pip install aiohttp") proxy_url = self._get_proxy_url() if not proxy_url: - return { - "final_response": "⚠️ Proxy URL not configured (GATEWAY_PROXY_URL or gateway.proxy_url)", - "messages": [], - "api_calls": 0, - "tools": [], - } + return self._proxy_error_result("⚠️ Proxy URL not configured (GATEWAY_PROXY_URL or gateway.proxy_url)") # Scope-aware read: the proxy key is a per-profile credential; under multiplex honor the # installed scope's verdict (Slack pattern for the unscoped default-profile loop). try: from agent.secret_scope import UnscopedSecretError, get_secret - try: proxy_key = (get_secret("GATEWAY_PROXY_KEY") or "").strip() except UnscopedSecretError: @@ -3586,182 +2540,117 @@ class GatewayTurnMixin: except Exception: proxy_key = os.getenv("GATEWAY_PROXY_KEY", "").strip() - def _run_still_current() -> bool: - if run_generation is None or not session_key: - return True - return self._is_session_run_current(session_key, run_generation) + _run_still_current = self._run_still_current_fn(session_key, run_generation) - # Build messages in OpenAI chat format. The remote api_server keeps continuity via - # X-Hermes-Session-Id and loads its own history, so send only the current message; if the - # remote has no history yet, include a compact text-only local history (remote replays tools). + def _stale_result(what: str) -> Dict[str, Any]: + logger.info( + "Discarding stale proxy %s for %s — generation %d is no longer current", + what, session_key or "?", run_generation or 0, + ) + return { + "final_response": "", "messages": [], "api_calls": 0, "tools": [], + "history_offset": len(history), "session_id": session_id, "response_previewed": False, + } + + # OpenAI chat format. The remote api_server keeps continuity via X-Hermes-Session-Id and + # loads its own history, so send only the current message plus a compact text-only local + # history for a remote that has none yet (remote replays tools). api_messages: List[Dict[str, str]] = [] - if context_prompt: api_messages.append({"role": "system", "content": context_prompt}) - for msg in history: role = msg.get("role") content = msg.get("content") if role in {"user", "assistant"} and content: api_messages.append({"role": role, "content": content}) - api_messages.append({"role": "user", "content": message}) - # HTTP headers --------------------------------------------------- headers: Dict[str, str] = {"Content-Type": "application/json"} if proxy_key: headers["Authorization"] = f"Bearer {proxy_key}" if session_id: headers["X-Hermes-Session-Id"] = session_id + body = {"model": "hermes-agent", "messages": api_messages, "stream": True} - body = { - "model": "hermes-agent", - "messages": api_messages, - "stream": True, - } - - # Set up platform streaming if available ------------------------- + # Platform streaming, if available. _stream_consumer = None _scfg = getattr(getattr(self, "config", None), "streaming", None) if _scfg is None: from gateway.config import StreamingConfig _scfg = StreamingConfig() - platform_key = _platform_config_key(source.platform) user_config = _load_gateway_config() from gateway.display_config import resolve_display_setting - _plat_streaming = resolve_display_setting( - user_config, platform_key, "streaming" - ) + _plat_streaming = resolve_display_setting(user_config, platform_key, "streaming") _streaming_enabled = ( - _scfg.enabled and _scfg.transport != "off" - if _plat_streaming is None - else bool(_plat_streaming) + _scfg.enabled and _scfg.transport != "off" if _plat_streaming is None else bool(_plat_streaming) ) - _thread_metadata: Optional[Dict[str, Any]] = self._thread_metadata_for_source(source, event_message_id) - if _streaming_enabled: try: from gateway.stream_consumer import GatewayStreamConsumer _adapter = self._adapter_for_source(source) if _adapter: - _consumer_cfg, _pause_typing_before_finalize = ( - self._build_stream_consumer_config( - source, _scfg, _adapter, - on_missing_cursor="fallback", - ) + _consumer_cfg, _pause_typing_before_finalize = self._build_stream_consumer_config( + source, _scfg, _adapter, on_missing_cursor="fallback", ) _stream_consumer = GatewayStreamConsumer( - adapter=_adapter, - chat_id=source.chat_id, - config=_consumer_cfg, - metadata=_thread_metadata, - on_before_finalize=_pause_typing_before_finalize, - initial_reply_to_id=event_message_id, - run_still_current=_run_still_current, + adapter=_adapter, chat_id=source.chat_id, config=_consumer_cfg, + metadata=_thread_metadata, on_before_finalize=_pause_typing_before_finalize, + initial_reply_to_id=event_message_id, run_still_current=_run_still_current, ) except Exception as _sc_err: logger.debug("Proxy: could not set up stream consumer: %s", _sc_err) + stream_task = asyncio.create_task(_stream_consumer.run()) if _stream_consumer else None - # Run the stream consumer task in the background - stream_task = None - if _stream_consumer: - stream_task = asyncio.create_task(_stream_consumer.run()) - - # Send typing indicator _adapter = self._adapter_for_source(source) if _adapter: with suppress(Exception): await _adapter.send_typing(source.chat_id, metadata=_thread_metadata) - # Make the HTTP request with SSE streaming ----------------------- full_response = "" _start = time.time() - try: _timeout = ClientTimeout(total=0, sock_read=1800) async with _AioClientSession(timeout=_timeout) as session: - async with session.post( - f"{proxy_url}/v1/chat/completions", - json=body, - headers=headers, - ) as resp: + async with session.post(f"{proxy_url}/v1/chat/completions", json=body, headers=headers) as resp: if resp.status != 200: error_text = await resp.text() - logger.warning( - "Proxy error (%d) from %s: %s", - resp.status, proxy_url, error_text[:500], - ) - return { - "final_response": f"⚠️ Proxy error ({resp.status}): {error_text[:300]}", - "messages": [], - "api_calls": 0, - "tools": [], - } + logger.warning("Proxy error (%d) from %s: %s", resp.status, proxy_url, error_text[:500]) + return self._proxy_error_result(f"⚠️ Proxy error ({resp.status}): {error_text[:300]}") - # Parse SSE stream buffer = "" async for chunk in resp.content.iter_any(): if not _run_still_current(): - logger.info( - "Discarding stale proxy stream for %s — generation %d is no longer current", - session_key or "?", - run_generation or 0, - ) - return { - "final_response": "", - "messages": [], - "api_calls": 0, - "tools": [], - "history_offset": len(history), - "session_id": session_id, - "response_previewed": False, - } - text = chunk.decode("utf-8", errors="replace") - buffer += text - - # Process complete SSE lines + return _stale_result("stream") + buffer += chunk.decode("utf-8", errors="replace") while "\n" in buffer: line, buffer = buffer.split("\n", 1) line = line.strip() - if not line: + if not line.startswith("data: "): continue - if line.startswith("data: "): - data = line[6:] - if data.strip() == "[DONE]": - break - try: - obj = json.loads(data) - choices = obj.get("choices", []) - if choices: - delta = choices[0].get("delta", {}) - content = delta.get("content", "") - if content: - full_response += content - if _stream_consumer: - _stream_consumer.on_delta(content) - except json.JSONDecodeError: - pass + data = line[6:] + if data.strip() == "[DONE]": + break + try: + choices = json.loads(data).get("choices", []) + except json.JSONDecodeError: + continue + content = choices[0].get("delta", {}).get("content", "") if choices else "" + if content: + full_response += content + if _stream_consumer: + _stream_consumer.on_delta(content) if len(buffer) > _GATEWAY_PROXY_SSE_BUFFER_MAX_CHARS: - raise ValueError( - "Proxy SSE stream exceeded max buffer size without a line boundary" - ) - + raise ValueError("Proxy SSE stream exceeded max buffer size without a line boundary") except asyncio.CancelledError: raise except Exception as e: logger.error("Proxy connection error to %s: %s", proxy_url, e) if not full_response: - return { - "final_response": f"⚠️ Proxy connection error: {e}", - "messages": [], - "api_calls": 0, - "tools": [], - } + return self._proxy_error_result(f"⚠️ Proxy connection error: {e}") # Partial response — return what we got finally: - # Finalize stream consumer if _stream_consumer: _stream_consumer.finish() if stream_task: @@ -3772,25 +2661,11 @@ class GatewayTurnMixin: _elapsed = time.time() - _start if not _run_still_current(): - logger.info( - "Discarding stale proxy result for %s — generation %d is no longer current", - session_key or "?", - run_generation or 0, - ) - return { - "final_response": "", - "messages": [], - "api_calls": 0, - "tools": [], - "history_offset": len(history), - "session_id": session_id, - "response_previewed": False, - } + return _stale_result("result") logger.info( "proxy response: url=%s session=%s time=%.1fs response=%d chars", proxy_url, (session_id or "")[:20], _elapsed, len(full_response), ) - return { "final_response": full_response or "(No response from remote agent)", "messages": [ @@ -3805,32 +2680,16 @@ class GatewayTurnMixin: } async def _run_agent( - self, - message: str, - context_prompt: str, - history: List[Dict[str, Any]], - source: SessionSource, - session_id: str, - session_key: str = None, - run_generation: Optional[int] = None, - _interrupt_depth: int = 0, - event_message_id: Optional[str] = None, - inbound_message_id: Optional[str] = None, - channel_prompt: Optional[str] = None, - moa_config: Optional[dict] = None, - persist_user_message: Optional[Any] = None, - persist_user_timestamp: Optional[float] = None, - persist_user_display_kind: Optional[str] = None, - message_type: Optional[str] = None, + self, message: str, context_prompt: str, history: List[Dict[str, Any]], + source: SessionSource, session_id: str, session_key: str = None, + run_generation: Optional[int] = None, _interrupt_depth: int = 0, + event_message_id: Optional[str] = None, inbound_message_id: Optional[str] = None, + channel_prompt: Optional[str] = None, moa_config: Optional[dict] = None, + persist_user_message: Optional[Any] = None, persist_user_timestamp: Optional[float] = None, + persist_user_display_kind: Optional[str] = None, message_type: Optional[str] = None, ) -> Dict[str, Any]: - """Profile-scoping wrapper around the agent run. - - Under multiplexing, run the turn inside ``_profile_runtime_scope`` so config/skills/memory - resolve to the source profile's home AND credentials come from its secret scope (never - process-global ``os.environ``). Transparent pass-through when multiplexing is off. - """ - from gateway.run import _profile_runtime_scope - if not getattr(getattr(self, "config", None), "multiplex_profiles", False): + """Profile-scoping wrapper around the agent run (pass-through when multiplexing is off).""" + with self._profile_scope_for_source(source): return await self._run_agent_inner( message, context_prompt, history, source, session_id, session_key=session_key, run_generation=run_generation, @@ -3839,60 +2698,38 @@ class GatewayTurnMixin: channel_prompt=channel_prompt, moa_config=moa_config, persist_user_message=persist_user_message, persist_user_timestamp=persist_user_timestamp, - persist_user_display_kind=persist_user_display_kind, - message_type=message_type, - ) - - profile_home = self._resolve_profile_home_for_source(source) - with _profile_runtime_scope(profile_home): - return await self._run_agent_inner( - message, context_prompt, history, source, session_id, - session_key=session_key, run_generation=run_generation, - _interrupt_depth=_interrupt_depth, event_message_id=event_message_id, - inbound_message_id=inbound_message_id, - channel_prompt=channel_prompt, moa_config=moa_config, - persist_user_message=persist_user_message, - persist_user_timestamp=persist_user_timestamp, - persist_user_display_kind=persist_user_display_kind, - message_type=message_type, + persist_user_display_kind=persist_user_display_kind, message_type=message_type, ) def _run_agent_display_settings(self, source: SessionSource) -> "GatewayRunner._RunAgentDisplay": """Resolve per-platform display, progress, status and streaming-surface settings for a turn.""" from gateway.run import ( - _gateway_platform_value, - _has_platform_display_override, - _load_gateway_config, + _gateway_platform_value, _has_platform_display_override, _load_gateway_config, _platform_config_key, ) + from agent.skill_utils import parse_config_string_list + from gateway.display_config import resolve_display_setting + from gateway.status_phrases import choose_status_phrase, resolve_status_phrase_catalog + user_config = _load_gateway_config() platform_key = _platform_config_key(source.platform) + enabled_toolsets = self._resolve_enabled_toolsets_for_source(user_config, source, platform_key) + disabled_toolsets = parse_config_string_list( + (user_config.get("agent") or {}).get("disabled_toolsets") + ) or None + # Per-platform display settings: display.platforms.., then display. + # global, then built-in platform defaults. + _display_cfg = user_config.get("display", {}) + if not isinstance(_display_cfg, dict): + _display_cfg = {} - enabled_toolsets = self._resolve_enabled_toolsets_for_source( - user_config, source, platform_key - ) - agent_cfg_local = user_config.get("agent") or {} - from agent.skill_utils import parse_config_string_list - - disabled_toolsets = parse_config_string_list(agent_cfg_local.get("disabled_toolsets")) or None - - display_config = user_config.get("display", {}) - if not isinstance(display_config, dict): - display_config = {} - - # Per-platform display settings via display_config: display.platforms.., then - # display. global, then built-in platform defaults. - from gateway.display_config import resolve_display_setting - - # Apply tool preview length config (0 = no limit) + # Tool preview length (0 = no limit) and friendly tool labels (default on), per-platform. try: from agent.display import set_tool_preview_max_len _tpl = resolve_display_setting(user_config, platform_key, "tool_preview_length", 0) set_tool_preview_max_len(int(_tpl) if _tpl else 0) except Exception: pass - - # Apply friendly tool labels config (default on) — per-platform aware try: from agent.display import set_friendly_tool_labels _ftl = resolve_display_setting(user_config, platform_key, "friendly_tool_labels", True) @@ -3900,49 +2737,31 @@ class GatewayTurnMixin: except Exception: pass - # Tool progress mode — resolved per-platform with env var fallback + # Tool progress mode — per-platform, with HERMES_TOOL_PROGRESS_MODE winning only when the + # config never set it. _resolved_tp = resolve_display_setting(user_config, platform_key, "tool_progress") _env_tp = os.getenv("HERMES_TOOL_PROGRESS_MODE") - _display_cfg = display_config if isinstance(display_config, dict) else {} - _platforms_cfg = _display_cfg.get("platforms") or {} - _platform_cfg = _platforms_cfg.get(platform_key) or {} + _platform_cfg = (_display_cfg.get("platforms") or {}).get(platform_key) or {} _legacy_tp_overrides = _display_cfg.get("tool_progress_overrides") or {} _tool_progress_configured = ( "tool_progress" in _display_cfg - or ( - isinstance(_platform_cfg, dict) - and "tool_progress" in _platform_cfg - ) - or ( - isinstance(_legacy_tp_overrides, dict) - and platform_key in _legacy_tp_overrides - ) + or (isinstance(_platform_cfg, dict) and "tool_progress" in _platform_cfg) + or (isinstance(_legacy_tp_overrides, dict) and platform_key in _legacy_tp_overrides) ) - progress_mode = ( - _env_tp - if _env_tp and not _tool_progress_configured - else (_resolved_tp or _env_tp or "all") - ) - # Tool progress grouping: "accumulate" (edit one bubble) or "separate" (one msg per tool) + progress_mode = _env_tp if _env_tp and not _tool_progress_configured else (_resolved_tp or _env_tp or "all") + # "accumulate" (edit one bubble) or "separate" (one msg per tool) progress_grouping = resolve_display_setting(user_config, platform_key, "tool_progress_grouping") or "accumulate" - from gateway.status_phrases import choose_status_phrase, resolve_status_phrase_catalog _generic_status_recent: List[str] = [] _generic_status_catalog = resolve_status_phrase_catalog(user_config, platform_key) def _display_surface_mode( - setting: str, - *, - default: bool = False, - require_platform_override_for: set[Any] | None = None, - allow_generic: bool = False, + setting: str, *, default: bool = False, + require_platform_override_for: set[Any] | None = None, allow_generic: bool = False, ) -> str: """Return off|raw|generic for a gateway visibility surface.""" if require_platform_override_for: current_platform = _gateway_platform_value(source.platform) - platform_only = { - _gateway_platform_value(item) - for item in require_platform_override_for - } + platform_only = {_gateway_platform_value(item) for item in require_platform_override_for} if ( current_platform in platform_only and not _has_platform_display_override(user_config, platform_key, setting) @@ -3956,197 +2775,124 @@ class GatewayTurnMixin: def _generic_status_phrase(kind: str, *, tool_name: str | None = None, preview: str | None = None, args: Any = None) -> str: try: return choose_status_phrase( - kind, - tool_name=tool_name, - preview=preview, - args=args, - recent=_generic_status_recent, - catalog=_generic_status_catalog, + kind, tool_name=tool_name, preview=preview, args=args, + recent=_generic_status_recent, catalog=_generic_status_catalog, ) except Exception as _phrase_err: logger.debug("generic status phrase selection failed: %s", _phrase_err) return "still on it" if kind in {"heartbeat", "waiting", "long_running", "status"} else "one sec" - # Disable tool progress for webhooks - they don't support message editing, - # so each progress line would be sent as a separate message. - from gateway.config import Platform - tool_progress_enabled = progress_mode not in {"off", "log"} and source.platform != Platform.WEBHOOK + + # Webhooks can't edit messages, so tool progress / log mode are off there (each progress + # line would be its own message). + is_webhook = source.platform == Platform.WEBHOOK + tool_progress_enabled = progress_mode not in {"off", "log"} and not is_webhook # Live working-state status for text-rendering typing indicators (Slack's assistant status - # line). Independent of tool_progress (Slack defaults it off; the status line is ephemeral). - # Rides the existing _keep_typing refresh — the callback only stores a phrase, no extra calls. - _live_status_mode = resolve_display_setting( - user_config, platform_key, "live_status", "full" - ) + # line). Independent of tool_progress; rides the existing _keep_typing refresh. + _live_status_mode = resolve_display_setting(user_config, platform_key, "live_status", "full") _live_status_adapter = self._adapter_for_source(source) - if not getattr(_live_status_adapter, "supports_status_text", False): + if not getattr(_live_status_adapter, "supports_status_text", False) or _live_status_mode == "off": _live_status_adapter = None - if _live_status_mode == "off": - _live_status_adapter = None - # "log" mode: tool calls are written to ~/.hermes/logs/tool_calls.log - # instead of the chat (#3459 / #3458). Gateway-only by design. - log_mode_enabled = progress_mode == "log" and source.platform != Platform.WEBHOOK - log_queue: "queue.Queue | None" = queue.Queue() if log_mode_enabled else None - # Natural assistant status messages are independent from tool progress and token streaming: - # tool_progress can stay quiet while users opt into concise mid-turn updates. + # "log" mode: tool calls go to ~/.hermes/logs/tool_calls.log instead of the chat. Gateway-only. + log_mode_enabled = progress_mode == "log" and not is_webhook + # Natural assistant status messages are independent from tool progress and token streaming. + # thinking_progress is independent too (same queue). Mattermost requires a per-platform + # opt-in for both: global scratch-text display leaks too easily into busy public threads. interim_assistant_messages_mode = _display_surface_mode( - "interim_assistant_messages", - default=True, - require_platform_override_for={Platform.MATTERMOST}, + "interim_assistant_messages", default=True, require_platform_override_for={Platform.MATTERMOST}, ) - interim_assistant_messages_enabled = ( - source.platform != Platform.WEBHOOK - and interim_assistant_messages_mode != "off" - ) - # thinking_progress is independent — if enabled, we need the progress queue even when - # tool_progress is off (thinking relay uses same infra). Mattermost requires a per-platform - # opt-in: global scratch-text display is too easy to leak into busy public threads. - _thinking_mode = _display_surface_mode( - "thinking_progress", - default=False, - require_platform_override_for={Platform.MATTERMOST}, - ) - _thinking_enabled = _thinking_mode != "off" - # Slack-native task cards: with the Slack adapter's opt-in, tool progress renders as native - # plan/task cards via chat.startStream, so the progress queue is needed even though Slack keeps - # text tool_progress off by default (requiring both flags would silently disable the feature). + interim_assistant_messages_enabled = not is_webhook and interim_assistant_messages_mode != "off" + _thinking_enabled = _display_surface_mode( + "thinking_progress", default=False, require_platform_override_for={Platform.MATTERMOST}, + ) != "off" + # Slack-native task cards render tool progress via chat.startStream, so the progress queue is + # needed even though Slack keeps text tool_progress off by default. _progress_adapter_for_native = self._adapter_for_source(source) _native_slack_task_cards = False - if ( - source.platform == Platform.SLACK - and _progress_adapter_for_native is not None - and hasattr(_progress_adapter_for_native, "native_task_cards_enabled") - ): + if source.platform == Platform.SLACK and hasattr(_progress_adapter_for_native, "native_task_cards_enabled"): try: - _native_slack_task_cards = bool( - _progress_adapter_for_native.native_task_cards_enabled() - ) + _native_slack_task_cards = bool(_progress_adapter_for_native.native_task_cards_enabled()) except Exception: logger.debug("Slack native task-card config check failed", exc_info=True) - needs_progress_queue = ( - tool_progress_enabled or _thinking_enabled or _native_slack_task_cards - ) return self._RunAgentDisplay( - user_config=user_config, - platform_key=platform_key, - enabled_toolsets=enabled_toolsets, - disabled_toolsets=disabled_toolsets, - resolve_display_setting=resolve_display_setting, - progress_mode=progress_mode, - progress_grouping=progress_grouping, + user_config=user_config, platform_key=platform_key, enabled_toolsets=enabled_toolsets, + disabled_toolsets=disabled_toolsets, resolve_display_setting=resolve_display_setting, + progress_mode=progress_mode, progress_grouping=progress_grouping, _display_surface_mode=_display_surface_mode, - tool_progress_enabled=tool_progress_enabled, - _live_status_mode=_live_status_mode, - _live_status_adapter=_live_status_adapter, - log_mode_enabled=log_mode_enabled, - log_queue=log_queue, + tool_progress_enabled=tool_progress_enabled, _live_status_mode=_live_status_mode, + _live_status_adapter=_live_status_adapter, log_mode_enabled=log_mode_enabled, + log_queue=queue.Queue() if log_mode_enabled else None, interim_assistant_messages_enabled=interim_assistant_messages_enabled, - _thinking_enabled=_thinking_enabled, - _native_slack_task_cards=_native_slack_task_cards, - needs_progress_queue=needs_progress_queue, + _thinking_enabled=_thinking_enabled, _native_slack_task_cards=_native_slack_task_cards, + needs_progress_queue=tool_progress_enabled or _thinking_enabled or _native_slack_task_cards, _generic_status_phrase=_generic_status_phrase, ) def _run_agent_build_turn_context( - self, - disp: "GatewayRunner._RunAgentDisplay", - AIAgent: Any, - *, - message: str, - context_prompt: str, - history: List[Dict[str, Any]], - source: SessionSource, - session_id: str, - session_key: Optional[str], - run_generation: Optional[int], - _interrupt_depth: int, - event_message_id: Optional[str], - inbound_message_id: Optional[str], - channel_prompt: Optional[str], - moa_config: Optional[dict], - persist_user_message: Optional[Any], - persist_user_timestamp: Optional[float], + self, disp: "GatewayRunner._RunAgentDisplay", AIAgent: Any, *, message: str, + context_prompt: str, history: List[Dict[str, Any]], source: SessionSource, session_id: str, + session_key: Optional[str], run_generation: Optional[int], _interrupt_depth: int, + event_message_id: Optional[str], inbound_message_id: Optional[str], + channel_prompt: Optional[str], moa_config: Optional[dict], + persist_user_message: Optional[Any], persist_user_timestamp: Optional[float], persist_user_display_kind: Optional[str], ) -> Tuple[TurnContext, TurnRunner, Any]: - """Build the progress queues / holders, the ``TurnContext`` and its ``TurnRunner``. + """Build the progress holders, the ``TurnContext`` and its ``TurnRunner``. Returns ``(turn_ctx, turn_runner, cleanup_adapter)``; the progress-bubble cleanup flags travel on ``turn_ctx._cleanup_progress`` / ``turn_ctx._cleanup_msg_ids``. """ from gateway.run import TurnRunner - def _run_still_current() -> bool: - if run_generation is None or not session_key: - return True - return self._is_session_run_current(session_key, run_generation) - - # Queue for progress messages (thread-safe) - progress_queue = queue.Queue() if disp.needs_progress_queue else None - last_tool = [None] # Mutable container for tracking in closure - last_progress_msg = [None] # Track last message for dedup - repeat_count = [0] # How many times the same message repeated - # True when the previous progress line was a terminal fenced code block — consecutive terminal - # calls then drop the repeated "💻 terminal" header and render back-to-back blocks. - last_was_terminal_block = [False] # Discord voice "verbal ack before tool calls": with the continuous mixer installed - # (discord.voice_fx.enabled), speak a short phrase over the idle bed on the FIRST tool call of - # the turn (from tool_start_callback, independent of the tool-progress text gate); once per turn. - _voice_ack_fired = [False] + # (discord.voice_fx.enabled), speak a short phrase on the FIRST tool call of the turn; once + # per turn. source.chat_id is the linked text channel; resolve the guild whose voice + # connection is bound to it (mirrors DiscordAdapter.play_tts). _voice_ack_guild: List[Optional[int]] = [None] if source.platform == Platform.DISCORD: _va = self.adapters.get(Platform.DISCORD) - # source.chat_id is the linked text channel; resolve the guild whose - # voice connection is bound to it (mirrors DiscordAdapter.play_tts). _vtc = getattr(_va, "_voice_text_channels", None) if isinstance(_vtc, dict) and hasattr(_va, "voice_mixer_active"): for _gid, _tc in _vtc.items(): if str(_tc) == str(source.chat_id) and _va.voice_mixer_active(_gid): _voice_ack_guild[0] = _gid break - _voice_ack_loop = asyncio.get_running_loop() - # voice_ack_callback extracted to TurnRunner.voice_ack_callback - # (published onto turn_ctx after the runner is constructed below). - - # Auto-cleanup of temporary progress bubbles (Telegram + any adapter that implements - # ``delete_message``). Failed runs skip cleanup so the bubbles remain as breadcrumbs. + # Auto-cleanup of temporary progress bubbles (any adapter implementing ``delete_message``; + # failed runs skip cleanup so the bubbles remain as breadcrumbs). getattr on the type, not + # attribute access: a fake adapter without delete_message means "can't delete", not a crash. _cleanup_progress = bool( disp.resolve_display_setting(disp.user_config, disp.platform_key, "cleanup_progress") ) _cleanup_adapter = self._adapter_for_source(source) if _cleanup_progress else None - # getattr, not attribute access — same duck-typed-adapter guard as the edit_message check in - # send_progress_messages: a fake adapter without delete_message means "can't delete", not a crash. _cleanup_delete = getattr(type(_cleanup_adapter), "delete_message", None) if _cleanup_adapter is not None else None if _cleanup_adapter is not None and ( - _cleanup_delete is None - or _cleanup_delete is BasePlatformAdapter.delete_message + _cleanup_delete is None or _cleanup_delete is BasePlatformAdapter.delete_message ): - # Adapter doesn't support deletion — silently disable. _cleanup_progress = False _cleanup_adapter = None - _cleanup_msg_ids: List[str] = [] - # First-touch onboarding latch: fires at most once per run, even if - # several tools exceed the threshold. - long_tool_hint_fired = [False] - _LONG_TOOL_THRESHOLD_S = 30.0 turn_ctx = TurnContext( source=source, - _run_still_current=_run_still_current, + _run_still_current=self._run_still_current_fn(session_key, run_generation), _live_status_adapter=disp._live_status_adapter, _live_status_mode=disp._live_status_mode, _thinking_enabled=disp._thinking_enabled, progress_mode=disp.progress_mode, progress_grouping=disp.progress_grouping, tool_progress_enabled=disp.tool_progress_enabled, - progress_queue=progress_queue, + progress_queue=queue.Queue() if disp.needs_progress_queue else None, log_queue=disp.log_queue, - last_progress_msg=last_progress_msg, - last_tool=last_tool, - last_was_terminal_block=last_was_terminal_block, - repeat_count=repeat_count, - long_tool_hint_fired=long_tool_hint_fired, - _LONG_TOOL_THRESHOLD_S=_LONG_TOOL_THRESHOLD_S, + # Mutable one-slot containers shared with the callbacks: last progress line (dedup), last + # tool, whether the previous line was a terminal fenced block (consecutive terminal calls + # drop the repeated header), repeat counter, first-touch long-tool onboarding latch. + last_progress_msg=[None], + last_tool=[None], + last_was_terminal_block=[False], + repeat_count=[0], + long_tool_hint_fired=[False], + _LONG_TOOL_THRESHOLD_S=30.0, _cleanup_progress=_cleanup_progress, - _cleanup_msg_ids=_cleanup_msg_ids, + _cleanup_msg_ids=[], message=message, AIAgent=AIAgent, resolve_display_setting=disp.resolve_display_setting, @@ -4157,9 +2903,9 @@ class GatewayTurnMixin: interim_assistant_messages_enabled=disp.interim_assistant_messages_enabled, needs_progress_queue=disp.needs_progress_queue, _native_slack_task_cards=disp._native_slack_task_cards, - _voice_ack_fired=_voice_ack_fired, + _voice_ack_fired=[False], _voice_ack_guild=_voice_ack_guild, - _voice_ack_loop=_voice_ack_loop, + _voice_ack_loop=asyncio.get_running_loop(), history=history, context_prompt=context_prompt, channel_prompt=channel_prompt, @@ -4175,75 +2921,69 @@ class GatewayTurnMixin: persist_user_display_kind=persist_user_display_kind, ) turn_runner = TurnRunner(self, turn_ctx) - # Callback invoked by agent on tool lifecycle events — extracted to - # TurnRunner.progress_callback (bound method, same signature). + # Agent tool-lifecycle callbacks live on the runner (bound methods, same signatures). turn_ctx.progress_callback = turn_runner.progress_callback turn_ctx.voice_ack_callback = turn_runner.voice_ack_callback turn_ctx.native_tool_start_callback = turn_runner.combined_tool_start_callback - turn_ctx.native_tool_complete_callback = ( - turn_runner.native_tool_complete_callback - ) + turn_ctx.native_tool_complete_callback = turn_runner.native_tool_complete_callback return turn_ctx, turn_runner, _cleanup_adapter + def _thread_metadata_for_progress( + self, source: SessionSource, event_message_id: Optional[str], _progress_thread_id: Any, + _relay_prospective_thread_id: Optional[str], + ) -> Optional[Dict[str, Any]]: + """Thread metadata for a progress-lane send; relay Discord auto-thread lane falls back to the reply anchor. + + The connector will auto-thread on the reply anchor (thread is born on its FIRST send), so + carrying it routes progress / status bubbles into the same thread as the final reply. + """ + if not _progress_thread_id: + metadata = None + elif _progress_thread_id == source.thread_id: + metadata = self._thread_metadata_for_source(source, event_message_id) + else: + metadata = self._thread_metadata_for_target( + source.platform, source.chat_id, _progress_thread_id, + chat_type=getattr(source, "chat_type", None), reply_to_message_id=event_message_id, + ) + if metadata is None and _relay_prospective_thread_id: + metadata = {"reply_to_message_id": event_message_id} + return metadata + def _run_agent_progress_threading( - self, - source: SessionSource, - event_message_id: Optional[str], - _native_slack_task_cards: bool, + self, source: SessionSource, event_message_id: Optional[str], _native_slack_task_cards: bool ) -> Tuple[Optional[dict], Optional[str], Any, Optional[str]]: """Resolve where progress bubbles are threaded. Returns ``(progress_metadata, progress_reply_to, progress_thread_id, relay_prospective_thread_id)``. + Threading is platform-specific: Slack DM threading needs the event_message_id fallback; + Telegram forum topics use message_thread_id and Hermes-created private DM topic lanes need + thread metadata plus a reply anchor; Feishu only honors reply_in_thread on a reply; others use + explicit source.thread_id only. Slack and Buzz honour the user's reply_in_thread opt-out: + never synthesise a thread for progress, or every later reply inherits it. """ from gateway.run import _non_conversational_metadata, _resolve_progress_thread_id - # Background task accumulating tool lines into one edited progress message. Threading metadata - # is platform-specific: Slack DM threading needs the event_message_id fallback; Telegram forum - # topics use message_thread_id and Hermes-created private DM topic lanes need thread metadata - # plus a reply anchor; Feishu only honors reply_in_thread on a reply, so topic progress replies - # to the triggering event; others use explicit source.thread_id only. Slack honours - # reply_in_thread=false: don't synthesise a thread for progress, or every later reply inherits it. + is_buzz = str(getattr(source.platform, "value", source.platform) or "").lower() == "buzz" _progress_reply_in_thread = True - if source.platform == Platform.SLACK: - _slack_adapter_for_progress = self._adapter_for_source(source) - if _slack_adapter_for_progress is not None: - try: - # Relay lane: adapter owns mode resolution (nested platforms.relay.extra.slack subset, - # flat-key fallback). Native lane: read the flat extra as before. - _mode_fn = getattr( - _slack_adapter_for_progress, - "_effective_reply_in_thread", - None, + _adapter = self._adapter_for_source(source) if source.platform == Platform.SLACK or is_buzz else None + if _adapter is not None: + try: + if is_buzz: + _progress_reply_in_thread = getattr(_adapter, "_reply_to_mode", "first") != "off" + else: + # Relay lane: adapter owns mode resolution (nested platforms.relay.extra.slack + # subset, flat-key fallback). Native lane: read the flat extra as before. + _mode_fn = getattr(_adapter, "_effective_reply_in_thread", None) + _progress_reply_in_thread = bool( + _mode_fn() if callable(_mode_fn) else _adapter.config.extra.get("reply_in_thread", True) ) - if callable(_mode_fn): - _progress_reply_in_thread = bool(_mode_fn()) - else: - _progress_reply_in_thread = bool( - _slack_adapter_for_progress.config.extra.get( - "reply_in_thread", True - ) - ) - except Exception: - _progress_reply_in_thread = True - elif str(getattr(source.platform, "value", source.platform) or "").lower() == "buzz": - # Buzz honours the same opt-out (reply_to_mode: off / extra.reply_in_thread: false): when the - # user asked for flat channel replies, progress must not synthesise a thread either. - _buzz_adapter_for_progress = self._adapter_for_source(source) - if _buzz_adapter_for_progress is not None: - try: - _progress_reply_in_thread = ( - getattr(_buzz_adapter_for_progress, "_reply_to_mode", "first") - != "off" - ) - except Exception: - _progress_reply_in_thread = True + except Exception: + _progress_reply_in_thread = True _progress_thread_id = _resolve_progress_thread_id( - source.platform, source.thread_id, event_message_id, - reply_in_thread=_progress_reply_in_thread, + source.platform, source.thread_id, event_message_id, reply_in_thread=_progress_reply_in_thread, ) - # Relay Discord auto-thread lane: a channel-initiating message has no thread_id at ingest - # (thread is born on the connector's FIRST send). The connector stamps prospective_thread_id - # (anchor id == the thread it will create); carry it as reply_to on the progress send so - # bubbles route into the SAME auto-thread instead of landing flat in the parent channel. + # Relay Discord auto-thread lane: a channel-initiating message has no thread_id at ingest; + # the connector stamps prospective_thread_id (anchor id == the thread it will create). _relay_prospective_thread_id = ( str(getattr(source, "prospective_thread_id", None)) if source.platform == Platform.DISCORD @@ -4252,45 +2992,26 @@ class GatewayTurnMixin: and not source.thread_id else None ) - _progress_metadata = ( - self._thread_metadata_for_source(source, event_message_id) - if _progress_thread_id == source.thread_id - else self._thread_metadata_for_target( - source.platform, - source.chat_id, - _progress_thread_id, - chat_type=getattr(source, "chat_type", None), - reply_to_message_id=event_message_id, - ) - ) if _progress_thread_id else None - if _progress_metadata is None and _relay_prospective_thread_id: - # No real thread yet, but the connector will auto-thread on the - # reply anchor; carry it so progress joins that thread. - _progress_metadata = {"reply_to_message_id": event_message_id} - _progress_metadata = _non_conversational_metadata(_progress_metadata, platform=source.platform) + _progress_metadata = _non_conversational_metadata( + self._thread_metadata_for_progress( + source, event_message_id, _progress_thread_id, _relay_prospective_thread_id, + ), + platform=source.platform, + ) if _native_slack_task_cards: - # chat.startStream in channels requires the recipient team/user - # pair; harmless extras elsewhere, so stamp them whenever known. + # chat.startStream in channels requires the recipient team/user pair; harmless elsewhere. _progress_metadata = dict(_progress_metadata or {}) if source.scope_id: _progress_metadata.setdefault("recipient_team_id", source.scope_id) _progress_metadata.setdefault("slack_team_id", source.scope_id) if source.user_id: _progress_metadata.setdefault("recipient_user_id", source.user_id) + # Buzz has no native thread_id; threading is always via reply-to the triggering event id + # (channel clutter otherwise), skipped when the user opted out of threaded replies. _progress_reply_to = ( event_message_id - if ( - source.platform in (Platform.FEISHU, Platform.MATTERMOST) - and source.thread_id - and event_message_id - ) - or ( - # Buzz has no native thread_id; threading is always via reply-to the triggering event id - # (channel clutter otherwise); skipped when the user opted out of threaded replies. - str(getattr(source.platform, "value", source.platform) or "").lower() == "buzz" - and event_message_id - and _progress_reply_in_thread - ) + if (source.platform in (Platform.FEISHU, Platform.MATTERMOST) and source.thread_id and event_message_id) + or (is_buzz and event_message_id and _progress_reply_in_thread) or _relay_prospective_thread_id else None ) @@ -4305,16 +3026,12 @@ class GatewayTurnMixin: if log_queue is None: return from logging.handlers import RotatingFileHandler - from agent.redact import RedactingFormatter log_dir = _hermes_home / "logs" log_dir.mkdir(parents=True, exist_ok=True) file_handler = RotatingFileHandler( - log_dir / "tool_calls.log", - maxBytes=5 * 1024 * 1024, - backupCount=3, - encoding="utf-8", + log_dir / "tool_calls.log", maxBytes=5 * 1024 * 1024, backupCount=3, encoding="utf-8", ) file_handler.setFormatter(RedactingFormatter("%(message)s")) tool_logger = logging.getLogger(f"hermes.tool_calls.{id(log_queue)}") @@ -4333,218 +3050,200 @@ class GatewayTurnMixin: except asyncio.CancelledError: pass finally: - # Drain remaining entries before closing so late tool calls - # from the final iteration aren't lost. - while True: - try: + # Drain remaining entries so late tool calls from the final iteration aren't lost. + with suppress(Exception): + while True: tool_logger.info("%s", log_queue.get_nowait()) - except queue.Empty: - break - except Exception: - break tool_logger.removeHandler(file_handler) - try: + with suppress(Exception): file_handler.flush() file_handler.close() - except Exception: - pass def _run_agent_status_thread_metadata( - self, - source: SessionSource, - event_message_id: Optional[str], - _progress_thread_id: Any, + self, source: SessionSource, event_message_id: Optional[str], _progress_thread_id: Any, _relay_prospective_thread_id: Optional[str], ) -> Optional[Dict[str, Any]]: - """Thread metadata for status / approval / stream sends (Feishu carries the reply anchor).""" + """Thread metadata for status / approval / stream sends. + + Feishu topics only keep messages inside the topic when sent via the reply API with + reply_in_thread=true; those paths usually only get metadata, so carry the triggering + message id as a Feishu-specific fallback. + """ if source.platform == Platform.FEISHU and source.thread_id and event_message_id: - # Feishu topics only keep messages inside the topic when they are sent via the reply API - # with reply_in_thread=true. Status/approval/stream paths usually only get metadata, so - # carry the triggering message id as a Feishu-specific fallback. - _status_thread_metadata: Optional[Dict[str, Any]] = { - "thread_id": _progress_thread_id, - "reply_to_message_id": event_message_id, - } - else: - _status_thread_metadata = ( - self._thread_metadata_for_source(source, event_message_id) - if _progress_thread_id == source.thread_id - else self._thread_metadata_for_target( - source.platform, - source.chat_id, - _progress_thread_id, - chat_type=getattr(source, "chat_type", None), - reply_to_message_id=event_message_id, - ) - ) if _progress_thread_id else None - if _status_thread_metadata is None and _relay_prospective_thread_id: - # Relay Discord auto-thread lane (see _progress_metadata): carry the reply anchor so - # status/interim bubbles route into the same connector-created thread as the final reply. - _status_thread_metadata = { - "reply_to_message_id": event_message_id - } - return _status_thread_metadata + return {"thread_id": _progress_thread_id, "reply_to_message_id": event_message_id} + return self._thread_metadata_for_progress( + source, event_message_id, _progress_thread_id, _relay_prospective_thread_id, + ) def _run_agent_start_streaming_tts( - self, - source: SessionSource, - message_type: Optional[str], - _status_thread_metadata: Optional[Dict[str, Any]], - streaming_tts_consumer_holder: list, + self, source: SessionSource, message_type: Optional[str], + _status_thread_metadata: Optional[Dict[str, Any]], streaming_tts_consumer_holder: list, ) -> None: - # Streaming TTS consumer setup. Created on the gateway event-loop thread (here), NOT inside - # run_sync's executor worker: the outer interrupt / finalisation paths reference the consumer - # via ``streaming_tts_consumer_holder[0]`` and would hit a cross-scope NameError. + """Start the streaming-TTS consumer for a voice-input turn on an auto-TTS chat. + + Created on the gateway event-loop thread (here), NOT inside run_sync's executor worker: the + outer interrupt / finalisation paths reference it via ``streaming_tts_consumer_holder[0]``. + An inactive consumer (no streaming provider) leaves the holder None so the whole-file + fallback path runs. + """ _stts_adapter = self._adapter_for_source(source) _is_voice_input = ( message_type is not None and str(getattr(message_type, "value", message_type)).lower() == "voice" ) - if ( - _stts_adapter is not None - and _is_voice_input - and _stts_adapter._should_auto_tts_for_chat(source.chat_id) - ): - try: - from gateway.streaming_tts_consumer import StreamingTTSConsumer - from tools.tts_tool import _load_tts_config - _tts_cfg = _load_tts_config() - _gateway_loop = self._gateway_loop or asyncio.get_event_loop() - _stts_consumer = StreamingTTSConsumer( - adapter=_stts_adapter, - chat_id=source.chat_id, - tts_config=_tts_cfg, - loop=_gateway_loop, - metadata=_status_thread_metadata, - ) - if _stts_consumer.active: - streaming_tts_consumer_holder[0] = _stts_consumer - _stts_consumer.start() - # else: consumer inactive (no streaming provider) — leave - # the holder as None so the whole-file fallback path runs. - except Exception as _stts_err: - logger.debug("Could not set up streaming TTS consumer: %s", _stts_err) + if _stts_adapter is None or not _is_voice_input or not _stts_adapter._should_auto_tts_for_chat(source.chat_id): + return + try: + from gateway.streaming_tts_consumer import StreamingTTSConsumer + from tools.tts_tool import _load_tts_config + _stts_consumer = StreamingTTSConsumer( + adapter=_stts_adapter, chat_id=source.chat_id, tts_config=_load_tts_config(), + loop=self._gateway_loop or asyncio.get_event_loop(), + metadata=_status_thread_metadata, + ) + if _stts_consumer.active: + streaming_tts_consumer_holder[0] = _stts_consumer + _stts_consumer.start() + except Exception as _stts_err: + logger.debug("Could not set up streaming TTS consumer: %s", _stts_err) async def _run_agent_stream_consumer_task(self, stream_consumer_holder: list) -> None: - """Wait for the stream consumer to be created, then run it.""" - for _ in range(200): # Up to 10s wait + """Wait (up to 10s) for the stream consumer to be created inside run_sync, then run it.""" + for _ in range(200): if stream_consumer_holder[0] is not None: await stream_consumer_holder[0].run() return await asyncio.sleep(0.05) async def _run_agent_track_agent( - self, - session_key: Optional[str], - run_generation: Optional[int], - agent_holder: list, + self, session_key: Optional[str], run_generation: Optional[int], agent_holder: list, ) -> None: - """Track this agent as running for the session (interrupt support) once it is created.""" - # Wait for agent to be created + """Track this agent as running for the session (interrupt support) once it is created. + + Only promote the sentinel to the real agent if this run is still current: if /stop or /new + bumped the generation while we were spinning up, leave the newer run's slot alone (the + stale-result check in _handle_message_with_agent discards us). + """ while agent_holder[0] is None: await asyncio.sleep(0.05) if not session_key: return - # Only promote the sentinel to the real agent if this run is still current. If /stop or - # /new bumped the generation while we were spinning up, leave the newer run's slot alone - # — we'll be discarded by the stale-result check in _handle_message_with_agent. - if run_generation is not None and not self._is_session_run_current( - session_key, run_generation - ): + if run_generation is not None and not self._is_session_run_current(session_key, run_generation): logger.info( "Skipping stale agent promotion for %s — generation %s is no longer current", - session_key or "", - run_generation, + session_key or "", run_generation, ) return self._session_state(session_key).turn.agent = agent_holder[0] if self._draining: self._update_runtime_status("draining") - async def _run_agent_monitor_for_interrupt( - self, - source: SessionSource, - session_key: Optional[str], - agent_holder: list, - _interrupt_detected: "asyncio.Event", - streaming_tts_consumer_holder: list, + async def _run_agent_fire_pending_interrupt( + self, adapter: Any, agent: Any, source: SessionSource, session_key: str, + _interrupt_detected: "asyncio.Event", streaming_tts_consumer_holder: list, *, + log_context: str, log: Callable[[], None], ) -> None: - # Monitor adapter interrupts (new messages). PRIMARY interrupt path for regular text: Level 1 - # (base.py) catches them before _handle_message(), so the Level 2 running_agent.interrupt() path - # never fires. The inactivity poll loop has a BACKUP check in case this task dies silently. + """Peek the adapter's pending event, transcribe voice, then signal the agent + abort streaming TTS. + + Peek WITHOUT consuming: the message must stay in _pending_messages for the post-run + ``_dequeue_pending_event()`` (full MessageEvent + media). Popping here races: the agent may + finish before checking _interrupt_requested, losing it. Audio is transcribed BEFORE signaling + so voice messages interrupt with the real transcript, not an empty string / path placeholder. + """ from gateway.run import _build_media_placeholder + _peek_event = adapter._pending_messages.get(session_key) + pending_text = None + if _peek_event is not None: + pending_text = _peek_event.text or "" + if self._pending_event_audio_paths(_peek_event): + pending_text, _ = await self._transcribe_and_echo_pending_voice( + _peek_event, adapter, source, pending_text, log_context=log_context, + metadata={"thread_id": source.thread_id} if source.thread_id else None, + ) + elif not pending_text and (getattr(_peek_event, "media_urls", None) or []): + pending_text = _build_media_placeholder(_peek_event) + log() + agent.interrupt(pending_text) + _interrupt_detected.set() + # Abort streaming TTS on barge-in. + _stts = streaming_tts_consumer_holder[0] + if _stts is not None: + _stts.abort("barge-in") + + async def _run_agent_monitor_for_interrupt( + self, source: SessionSource, session_key: Optional[str], agent_holder: list, + _interrupt_detected: "asyncio.Event", streaming_tts_consumer_holder: list, + ) -> None: + """Poll the adapter for interrupts (new messages) every 200ms and signal the agent. + + PRIMARY interrupt path for regular text: Level 1 (base.py) catches them before + _handle_message(), so the Level 2 running_agent.interrupt() path never fires. The inactivity + poll loop has a BACKUP check in case this task dies silently. Interrupts are keyed by + session_key (build_session_key output), NOT source.chat_id. + """ if not session_key: return - while True: - await asyncio.sleep(0.2) # Check every 200ms + await asyncio.sleep(0.2) try: - # Re-resolve adapter each iteration so reconnects don't - # leave us holding a stale reference. + # Re-resolve the adapter each iteration so reconnects don't leave a stale reference. _adapter = self._adapter_for_source(source) if not _adapter: continue - # Must use session_key (build_session_key output), NOT source.chat_id: the adapter - # stores interrupt events under the full session key. if hasattr(_adapter, 'has_pending_interrupt') and _adapter.has_pending_interrupt(session_key): agent = agent_holder[0] if agent: - # Peek WITHOUT consuming: the message must stay in _pending_messages for the - # post-run _dequeue_pending_event() (full MessageEvent + media). Popping here - # races: the agent may finish before checking _interrupt_requested, losing it. - _peek_event = _adapter._pending_messages.get(session_key) - pending_text = None - if _peek_event is not None: - pending_text = _peek_event.text or "" - # Transcribe audio BEFORE signaling the agent, so voice messages interrupt - # with the real transcript, not an empty string / file-path placeholder. - _media_urls = getattr(_peek_event, "media_urls", None) or [] - if self._pending_event_audio_paths(_peek_event): - pending_text, _ = await self._transcribe_and_echo_pending_voice( - _peek_event, - _adapter, - source, - pending_text, - log_context="Voice-interrupt", - metadata={"thread_id": source.thread_id} if source.thread_id else None, - ) - elif not pending_text and _media_urls: - pending_text = _build_media_placeholder(_peek_event) - logger.debug("Interrupt detected from adapter, signaling agent...") - agent.interrupt(pending_text) - _interrupt_detected.set() - # Abort streaming TTS on barge-in (#60671). - _stts = streaming_tts_consumer_holder[0] - if _stts is not None: - _stts.abort("barge-in") + await self._run_agent_fire_pending_interrupt( + _adapter, agent, source, session_key, _interrupt_detected, + streaming_tts_consumer_holder, log_context="Voice-interrupt", + log=lambda: logger.debug("Interrupt detected from adapter, signaling agent..."), + ) break except asyncio.CancelledError: raise except Exception as _mon_err: logger.debug("monitor_for_interrupt error (will retry): %s", _mon_err) + async def _run_agent_backup_interrupt_check( + self, source: SessionSource, session_key: Optional[str], agent_holder: list, + _interrupt_detected: "asyncio.Event", interrupt_monitor: "asyncio.Task", + streaming_tts_consumer_holder: list, + ) -> None: + """Backup interrupt check: if the monitor task died or missed the interrupt, catch it here.""" + if _interrupt_detected.is_set() or not session_key: + return + _backup_adapter = self._adapter_for_source(source) + _backup_agent = agent_holder[0] + if (_backup_adapter and _backup_agent + and hasattr(_backup_adapter, 'has_pending_interrupt') + and _backup_adapter.has_pending_interrupt(session_key)): + await self._run_agent_fire_pending_interrupt( + _backup_adapter, _backup_agent, source, session_key, _interrupt_detected, + streaming_tts_consumer_holder, + log_context="Voice-backup-interrupt", + log=lambda: logger.info( + "Backup interrupt detected for session %s (monitor task state: %s)", + session_key, "done" if interrupt_monitor.done() else "running", + ), + ) + @staticmethod - def _run_agent_stream_confirmed_final_delivery( - consumer, - final_text: str, - *, - previewed: bool = False, - ) -> bool: - """Return True only when the actual final reply reached the user.""" + def _run_agent_stream_confirmed_final_delivery(consumer, final_text: str, *, previewed: bool = False) -> bool: + """Return True only when the actual final reply reached the user. + + A successful finalize call is not proof the *content* was final: the edit may carry only the + last preview snapshot. Reconcile against the recorded turn-final payload: only a demonstrable + mismatch (False, incl. payload-less split delivery) overrides the flag; None keeps legacy + trust so timeout dedup isn't regressed. + """ if consumer is None: return False if getattr(consumer, "final_response_sent", False): - # A successful finalize call is not proof the *content* was final: the edit may carry - # only the last preview snapshot. Reconcile against the recorded turn-final payload: - # only a demonstrable mismatch (False, incl. payload-less split delivery) overrides - # the flag; None keeps legacy trust so timeout dedup isn't regressed. matcher = getattr(consumer, "delivered_final_matches", None) if callable(matcher): - try: + with suppress(Exception): if matcher(final_text) is False: return False - except Exception: - pass return True if previewed: has_delivered_text = getattr(consumer, "has_delivered_text", None) @@ -4556,30 +3255,26 @@ class GatewayTurnMixin: return False def _run_agent_start_turn_worker( - self, - turn_ctx: TurnContext, - run_sync: Callable[[], Any], - agent_holder: list, - session_id: str, - session_key: Optional[str], - run_generation: Optional[int], + self, turn_ctx: TurnContext, run_sync: Callable[[], Any], agent_holder: list, + session_id: str, session_key: Optional[str], run_generation: Optional[int], ) -> "GatewayRunner._RunAgentWorker": - """Schedule ``run_sync`` on the executor plus the inactivity watchdog thread.""" + """Schedule ``run_sync`` on the executor plus the inactivity watchdog thread. + + *Inactivity* timeout, not wall-clock: the agent may run for hours while actively calling + tools / streaming, but a hung API call or stuck tool is killed. agent.gateway_timeout / + HERMES_AGENT_TIMEOUT (env wins); default 1800s; 0 = unlimited. The daemon watchdog is + independent of asyncio: cgroup memory reclaim can starve the loop that runs the normal + timeout poll, and cleanup must not wait for the loop to recover. + """ from gateway.run import _float_env, _watch_gateway_turn_inactivity - # Thread pool so we don't block. *Inactivity* timeout, not wall-clock: the agent may run for - # hours while actively calling tools / streaming, but a hung API call or stuck tool is killed. - # agent.gateway_timeout / HERMES_AGENT_TIMEOUT (env wins); default 1800s; 0 = unlimited. + from tools.process_registry import process_registry _agent_timeout_raw = _float_env("HERMES_AGENT_TIMEOUT", 1800) _agent_timeout = _agent_timeout_raw if _agent_timeout_raw > 0 else None _agent_warning_raw = _float_env("HERMES_AGENT_TIMEOUT_WARNING", 900) _agent_warning = _agent_warning_raw if _agent_warning_raw > 0 else None - # A background=true process intentionally survives a successful turn, so capture - # existing IDs and reap only children created by THIS turn if it times out. The daemon - # watchdog is independent of asyncio: cgroup memory reclaim can starve the loop that - # runs the normal timeout poll, and cleanup must not wait for the loop to recover. - from tools.process_registry import process_registry - + # A background=true process intentionally survives a successful turn, so capture existing IDs + # and reap only children created by THIS turn if it times out. _turn_task_id = session_id or "" _turn_process_baseline = process_registry.snapshot_running_ids(_turn_task_id) turn_ctx.process_task_id = _turn_task_id @@ -4590,10 +3285,9 @@ class GatewayTurnMixin: # task_id is session-scoped, not turn-scoped: gate the eventual reap on this exact claim still # being current, so a replacement turn on the same session that starts before the watchdog # fires doesn't get its own fresh process killed by this turn's stale baseline. - _turn_run_generation = run_generation _turn_is_current = ( - (lambda: self._is_session_run_current(session_key, _turn_run_generation)) - if _turn_run_generation is not None + (lambda: self._is_session_run_current(session_key, run_generation)) + if run_generation is not None else (lambda: True) ) @@ -4603,9 +3297,9 @@ class GatewayTurnMixin: finally: _turn_worker_done.set() # `.turn.agent` is only reset to _AGENT_PENDING_SENTINEL when the *next* turn is - # claimed, so this agent stays reachable from _interrupt_and_clear_session() - # until then. Clearing ownership markers the instant our worker finishes means a - # /stop on the finished turn no longer reaps background work it left running. + # claimed, so this agent stays reachable from _interrupt_and_clear_session() until + # then. Clearing ownership markers the instant our worker finishes means a /stop on + # the finished turn no longer reaps background work it left running. _finished_agent = agent_holder[0] if agent_holder else None if _finished_agent is not None: _finished_agent._gateway_turn_process_task_id = "" @@ -4628,11 +3322,10 @@ class GatewayTurnMixin: name=f"gateway-turn-watchdog-{_turn_task_id[:12]}", daemon=True, ).start() - _executor_task = asyncio.ensure_future( - self._run_in_executor_with_context(_run_sync_with_timeout_lifecycle) - ) return self._RunAgentWorker( - executor_task=_executor_task, + executor_task=asyncio.ensure_future( + self._run_in_executor_with_context(_run_sync_with_timeout_lifecycle) + ), agent_timeout=_agent_timeout, agent_warning=_agent_warning, task_id=_turn_task_id, @@ -4643,130 +3336,49 @@ class GatewayTurnMixin: is_current=_turn_is_current, ) - async def _run_agent_backup_interrupt_check( - self, - source: SessionSource, - session_key: Optional[str], - agent_holder: list, - _interrupt_detected: "asyncio.Event", - interrupt_monitor: "asyncio.Task", - streaming_tts_consumer_holder: list, - ) -> None: - """Backup interrupt check: if the monitor task died or missed the interrupt, catch it here.""" - from gateway.run import _build_media_placeholder - if not _interrupt_detected.is_set() and session_key: - _backup_adapter = self._adapter_for_source(source) - _backup_agent = agent_holder[0] - if (_backup_adapter and _backup_agent - and hasattr(_backup_adapter, 'has_pending_interrupt') - and _backup_adapter.has_pending_interrupt(session_key)): - _bp_event = _backup_adapter._pending_messages.get(session_key) - _bp_text = _bp_event.text if _bp_event else None - if _bp_event is not None: - _bp_media_urls = getattr(_bp_event, "media_urls", None) or [] - if self._pending_event_audio_paths(_bp_event): - _bp_text, _ = await self._transcribe_and_echo_pending_voice( - _bp_event, - _backup_adapter, - source, - _bp_text or "", - log_context="Voice-backup-interrupt", - metadata={"thread_id": source.thread_id} if source.thread_id else None, - ) - elif not _bp_text and _bp_media_urls: - _bp_text = _build_media_placeholder(_bp_event) - logger.info( - "Backup interrupt detected for session %s " - "(monitor task state: %s)", - session_key, - "done" if interrupt_monitor.done() else "running", - ) - _backup_agent.interrupt(_bp_text) - _interrupt_detected.set() - # Abort streaming TTS on barge-in (#60671). - _stts = streaming_tts_consumer_holder[0] - if _stts is not None: - _stts.abort("barge-in") + @staticmethod + def _agent_activity_summary(agent: Any) -> dict: + """``agent.get_activity_summary()`` or ``{}`` when unavailable / failing.""" + if agent and hasattr(agent, "get_activity_summary"): + with suppress(Exception): + return agent.get_activity_summary() + return {} async def _run_agent_await_turn_worker( - self, - worker: "GatewayRunner._RunAgentWorker", - *, - source: SessionSource, - session_key: Optional[str], - agent_holder: list, - result_holder: list, - tools_holder: list, - _interrupt_detected: "asyncio.Event", - interrupt_monitor: "asyncio.Task", - streaming_tts_consumer_holder: list, - _status_thread_metadata: Optional[Dict[str, Any]], + self, worker: "GatewayRunner._RunAgentWorker", *, source: SessionSource, + session_key: Optional[str], agent_holder: list, result_holder: list, tools_holder: list, + _interrupt_detected: "asyncio.Event", interrupt_monitor: "asyncio.Task", + streaming_tts_consumer_holder: list, _status_thread_metadata: Optional[Dict[str, Any]], ) -> Any: """Poll the executor future (inactivity timeout + backup interrupt checks); return its result. - On inactivity timeout the result is a synthetic failed run dict carrying the diagnostic. + Polls the agent's activity tracker (updated by _touch_activity() on every tool call, API + call and stream delta) every few seconds; with an unlimited timeout it still polls for the + backup interrupt check in case monitor_for_interrupt() silently died. On inactivity timeout + the result is a synthetic failed run dict carrying the diagnostic. """ from gateway.run import ( - _INTERRUPT_REASON_TIMEOUT, - _abandon_timed_out_gateway_turn, - _interim_metadata, + _INTERRUPT_REASON_TIMEOUT, _abandon_timed_out_gateway_turn, _interim_metadata, request_hard_interrupt, ) _warning_fired = False _inactivity_timeout = False - _POLL_INTERVAL = 5.0 - - if worker.agent_timeout is None: - # Unlimited — still poll periodically for backup interrupt - # detection in case monitor_for_interrupt() silently died. - response = None - while True: - done, _ = await asyncio.wait( - {worker.executor_task}, timeout=_POLL_INTERVAL - ) - if done: - response = worker.executor_task.result() - break - # Backup interrupt check: if the monitor task died or - # missed the interrupt, catch it here. - await self._run_agent_backup_interrupt_check( - source, - session_key, - agent_holder, - _interrupt_detected, - interrupt_monitor, - streaming_tts_consumer_holder, - ) - - else: - # Poll the agent's built-in activity tracker (updated by _touch_activity() on every tool - # call, API call, and stream delta) every few seconds. - response = None - while True: - done, _ = await asyncio.wait( - {worker.executor_task}, timeout=_POLL_INTERVAL - ) - if done: - # Prefer the real result when the worker finished even if the watchdog fired in - # the same window: the completed run already persisted its reply, so the "agent - # inactive" diagnostic would contradict the stored transcript. - response = worker.executor_task.result() - break + response = None + while True: + done, _ = await asyncio.wait({worker.executor_task}, timeout=5.0) + if done: + # Prefer the real result even if the watchdog fired in the same window: the completed + # run already persisted its reply, so the "agent inactive" diagnostic would contradict + # the stored transcript. + response = worker.executor_task.result() + break + if worker.agent_timeout is not None: if worker.timeout_fired.is_set(): _inactivity_timeout = True break - # Agent still running — check inactivity. - _agent_ref = agent_holder[0] - _idle_secs = 0.0 - if _agent_ref and hasattr(_agent_ref, "get_activity_summary"): - try: - _act = _agent_ref.get_activity_summary() - _idle_secs = _act.get("seconds_since_activity", 0.0) - except Exception: - pass + _idle_secs = self._agent_activity_summary(agent_holder[0]).get("seconds_since_activity", 0.0) # Staged warning: fire once before escalating to full timeout. - if (not _warning_fired and worker.agent_warning is not None - and _idle_secs >= worker.agent_warning): + if not _warning_fired and worker.agent_warning is not None and _idle_secs >= worker.agent_warning: _warning_fired = True _warn_adapter = self._adapter_for_source(source) if _warn_adapter: @@ -4774,8 +3386,7 @@ class GatewayTurnMixin: _remaining_mins = int((worker.agent_timeout - worker.agent_warning) // 60) or 1 try: await _warn_adapter.send( - source.chat_id, - f"⚠️ No activity for {_elapsed_warn} min. " + source.chat_id, f"⚠️ No activity for {_elapsed_warn} min. " f"If the agent does not respond soon, it will " f"be timed out in {_remaining_mins} min. " f"You can continue waiting or use /reset.", @@ -4800,170 +3411,141 @@ class GatewayTurnMixin: daemon=True, ).start() break - # Backup interrupt check (same as unlimited path). - await self._run_agent_backup_interrupt_check( - source, - session_key, - agent_holder, - _interrupt_detected, - interrupt_monitor, - streaming_tts_consumer_holder, - ) - - if _inactivity_timeout: - # Build a diagnostic summary from the agent's activity tracker. - _timed_out_agent = agent_holder[0] - _activity = {} - if _timed_out_agent and hasattr(_timed_out_agent, "get_activity_summary"): - with suppress(Exception): - _activity = _timed_out_agent.get_activity_summary() - - _last_desc = _activity.get("last_activity_desc", "unknown") - _secs_ago = _activity.get("seconds_since_activity", 0) - _cur_tool = _activity.get("current_tool") - _iter_n = _activity.get("api_call_count", 0) - _iter_max = _activity.get("max_iterations", 0) - - logger.error( - "Agent idle for %.0fs (timeout %.0fs) in session %s " - "| last_activity=%s | iteration=%s/%s | tool=%s", - _secs_ago, worker.agent_timeout, session_key, - _last_desc, _iter_n, _iter_max, - _cur_tool or "none", + await self._run_agent_backup_interrupt_check( + source, session_key, agent_holder, _interrupt_detected, interrupt_monitor, + streaming_tts_consumer_holder, ) - # Interrupt the agent if it's still running so the thread - # pool worker is freed. - if _timed_out_agent: - request_hard_interrupt(_timed_out_agent, _INTERRUPT_REASON_TIMEOUT) - - _timeout_mins = int(worker.agent_timeout // 60) or 1 - - # Construct a user-facing message with diagnostic context. - _diag_lines = [ - f"⏱️ Agent inactive for {_timeout_mins} min — no tool calls " - f"or API responses." - ] - if _cur_tool: - _diag_lines.append( - f"The agent appears stuck on tool `{_cur_tool}` " - f"({_secs_ago:.0f}s since last activity, " - f"iteration {_iter_n}/{_iter_max})." - ) - else: - _diag_lines.append( - f"Last activity: {_last_desc} ({_secs_ago:.0f}s ago, " - f"iteration {_iter_n}/{_iter_max}). " - "The agent may have been waiting on an API response." - ) + if not _inactivity_timeout: + return response + # Diagnostic summary from the agent's activity tracker. + _timed_out_agent = agent_holder[0] + _activity = self._agent_activity_summary(_timed_out_agent) + _last_desc = _activity.get("last_activity_desc", "unknown") + _secs_ago = _activity.get("seconds_since_activity", 0) + _cur_tool = _activity.get("current_tool") + _iter_n = _activity.get("api_call_count", 0) + _iter_max = _activity.get("max_iterations", 0) + logger.error( + "Agent idle for %.0fs (timeout %.0fs) in session %s " + "| last_activity=%s | iteration=%s/%s | tool=%s", + _secs_ago, worker.agent_timeout, session_key, _last_desc, _iter_n, _iter_max, + _cur_tool or "none", + ) + # Interrupt the agent if it's still running so the thread pool worker is freed. + if _timed_out_agent: + request_hard_interrupt(_timed_out_agent, _INTERRUPT_REASON_TIMEOUT) + _timeout_mins = int(worker.agent_timeout // 60) or 1 + _diag_lines = [ + f"⏱️ Agent inactive for {_timeout_mins} min — no tool calls or API responses." + ] + if _cur_tool: _diag_lines.append( - "To increase the limit, set agent.gateway_timeout in config.yaml " - "(value in seconds, 0 = no limit) and restart the gateway.\n" - "Try again, or use /reset to start fresh." + f"The agent appears stuck on tool `{_cur_tool}` " + f"({_secs_ago:.0f}s since last activity, " + f"iteration {_iter_n}/{_iter_max})." ) - - response = { - "final_response": "\n".join(_diag_lines), - "messages": result_holder[0].get("messages", []) if result_holder[0] else [], - "api_calls": _iter_n, - "tools": tools_holder[0] or [], - "history_offset": 0, - "failed": True, - } - return response + else: + _diag_lines.append( + f"Last activity: {_last_desc} ({_secs_ago:.0f}s ago, " + f"iteration {_iter_n}/{_iter_max}). " + "The agent may have been waiting on an API response." + ) + _diag_lines.append( + "To increase the limit, set agent.gateway_timeout in config.yaml " + "(value in seconds, 0 = no limit) and restart the gateway.\n" + "Try again, or use /reset to start fresh." + ) + return { + "final_response": "\n".join(_diag_lines), + "messages": result_holder[0].get("messages", []) if result_holder[0] else [], + "api_calls": _iter_n, + "tools": tools_holder[0] or [], + "history_offset": 0, + "failed": True, + } def _run_agent_evict_on_fallback( self, session_key: Optional[str], agent_holder: list, result_holder: list, ) -> None: - # Persist fallback-model switches so /model shows the actually-active model. Skip - # eviction when the run failed — evicting forces MCP reinit on the next message for no - # benefit (bad model → fallback → evict → recreate → same 400 loop burning CPU). + """Evict the cached agent when a fallback model activated on a successful run. + + So /model shows the actually-active model and the next message retries the primary. Skip + when the run failed — evicting forces MCP reinit on the next message for no benefit (bad + model → fallback → evict → recreate → same 400 loop burning CPU). + """ from gateway.run import _resolve_gateway_model _agent = agent_holder[0] _result_for_fb = result_holder[0] _run_failed = _result_for_fb.get("failed") if _result_for_fb else False - if _agent is not None and hasattr(_agent, 'model') and not _run_failed: - _cfg_model = _resolve_gateway_model() - # Normalize _cfg_model as AIAgent.__init__ does so a vendor-prefixed config value - # matches the agent's stripped model on native providers — otherwise the cached agent - # is evicted every turn, destroying prompt caching. Aggregators keep the vendor slug. - try: - from hermes_cli.model_normalize import ( - _AGGREGATOR_PROVIDERS, - normalize_model_for_provider, - ) - _agent_provider = getattr(_agent, 'provider', '') or '' - if _agent_provider and _agent_provider not in _AGGREGATOR_PROVIDERS: - _cfg_model = normalize_model_for_provider(_cfg_model, _agent_provider) - except Exception: - pass - if _agent.model != _cfg_model and not self._is_intentional_model_switch(session_key, _agent.model): - # Fallback activated on a successful run — evict cached - # agent so the next message retries the primary model. - self._evict_cached_agent(session_key) + if _agent is None or not hasattr(_agent, 'model') or _run_failed: + return + _cfg_model = _resolve_gateway_model() + # Normalize _cfg_model as AIAgent.__init__ does so a vendor-prefixed config value matches + # the agent's stripped model on native providers — otherwise the cached agent is evicted + # every turn, destroying prompt caching. Aggregators keep the vendor slug. + try: + from hermes_cli.model_normalize import _AGGREGATOR_PROVIDERS, normalize_model_for_provider + _agent_provider = getattr(_agent, 'provider', '') or '' + if _agent_provider and _agent_provider not in _AGGREGATOR_PROVIDERS: + _cfg_model = normalize_model_for_provider(_cfg_model, _agent_provider) + except Exception: + pass + if _agent.model != _cfg_model and not self._is_intentional_model_switch(session_key, _agent.model): + self._evict_cached_agent(session_key) async def _run_agent_finalize_streaming_tts( - self, - streaming_tts_consumer_holder: list, - adapter: Any, - session_key: Optional[str], + self, streaming_tts_consumer_holder: list, adapter: Any, session_key: Optional[str], run_generation: Optional[int], ) -> None: - # Finalize the streaming-TTS consumer. finish() runs on the outer event-loop thread so - # early returns from run_sync are also finalised. wait_complete() drains queued audio; - # on timeout abort unconditionally — if audio was audible keep suppression (no replay - # from the start); if not, the whole-file fallback is permitted. + """Finalize the streaming-TTS consumer on the outer event-loop thread. + + finish() here covers early returns from run_sync too. wait_complete() drains queued audio; + on timeout abort unconditionally to free the consumer task — audible streams retain + whole-file suppression (no replay from the start); silent streams stay eligible for the + whole-file fallback. + """ _stts = streaming_tts_consumer_holder[0] - if _stts is not None: - _stts.finish() - try: - await _stts.wait_complete(timeout=10.0) - except Exception as _stts_done_err: - logger.debug("streaming TTS wait_complete error: %s", _stts_done_err) - if not _stts.done: - # Timeout before or after audible audio: abort to free the consumer task. Audible - # streams retain suppression; silent streams stay eligible for whole-file fallback. - _stts.abort("streaming TTS finalisation timeout") - await _stts.wait_complete(timeout=2.0) - if _stts.suppress_whole_file and adapter is not None: - _mark_turn = getattr(adapter, "_mark_streaming_tts_completed_turn", None) - if callable(_mark_turn): - _mark_turn(session_key, run_generation) + if _stts is None: + return + _stts.finish() + try: + await _stts.wait_complete(timeout=10.0) + except Exception as _stts_done_err: + logger.debug("streaming TTS wait_complete error: %s", _stts_done_err) + if not _stts.done: + _stts.abort("streaming TTS finalisation timeout") + await _stts.wait_complete(timeout=2.0) + if _stts.suppress_whole_file and adapter is not None: + _mark_turn = getattr(adapter, "_mark_streaming_tts_completed_turn", None) + if callable(_mark_turn): + _mark_turn(session_key, run_generation) async def _run_agent_drain_pending( - self, - result: Any, - adapter: Any, - source: SessionSource, - session_key: Optional[str], + self, result: Any, adapter: Any, source: SessionSource, session_key: Optional[str] ) -> Tuple[Any, Optional[str]]: - """Dequeue the adapter's pending / interrupt / leftover-steer follow-up. + """Dequeue the adapter's pending / interrupt / leftover-steer follow-up as ``(pending_event, pending)``. - Returns ``(pending_event, pending)``. + Keyed by session_key (not source.chat_id) to match the adapter's storage keys. """ from gateway.run import ( - _build_media_placeholder, - _dequeue_pending_event, - _is_control_interrupt_message, + _build_media_placeholder, _dequeue_pending_event, _is_control_interrupt_message ) - # Get pending message from adapter. - # Use session_key (not source.chat_id) to match adapter's storage keys. pending_event = None pending = None if result and adapter and session_key: pending_event = _dequeue_pending_event(adapter, session_key) - # /queue overflow: after consuming the adapter's "next-up" slot, promote the next - # queued event into it so the recursive run's drain will see it. Keeping the slot - # occupied for the whole FIFO chain preserves order and makes a mid-chain /queue - # route to overflow instead of jumping the queue. + # /queue overflow: after consuming the adapter's "next-up" slot, promote the next queued + # event into it so the recursive run's drain will see it. Keeping the slot occupied for + # the whole FIFO chain preserves order and makes a mid-chain /queue route to overflow + # instead of jumping the queue. pending_event = self._promote_queued_event(session_key, adapter, pending_event) if result.get("interrupted") and not pending_event and result.get("interrupt_message"): interrupt_message = result.get("interrupt_message") if _is_control_interrupt_message(interrupt_message): logger.info( "Ignoring control interrupt message for session %s: %s", - session_key or "?", - interrupt_message, + session_key or "?", interrupt_message, ) else: pending = interrupt_message @@ -4971,18 +3553,12 @@ class GatewayTurnMixin: # Transcribe audio on the dequeued event BEFORE it becomes the next user turn, so # queued/interrupting voice messages drain with the real transcript, not a file path. _pending_text = pending_event.text or "" - _media_urls = getattr(pending_event, "media_urls", None) or [] if self._pending_event_audio_paths(pending_event): pending, _ = await self._transcribe_and_echo_pending_voice( - pending_event, - adapter, - source, - _pending_text, - log_context="Voice-drain", + pending_event, adapter, source, _pending_text, log_context="Voice-drain", metadata={"thread_id": source.thread_id} if source.thread_id else None, ) - if not pending: - pending = _build_media_placeholder(pending_event) + pending = pending or _build_media_placeholder(pending_event) else: pending = _pending_text or _build_media_placeholder(pending_event) if pending: @@ -4990,57 +3566,42 @@ class GatewayTurnMixin: # Leftover /steer: a steer arriving after the last tool batch (e.g. during the final API # call) comes back in result["pending_steer"]; deliver it as the next user turn, not drop it. - if result and not pending and not pending_event: - _leftover_steer = result.get("pending_steer") - if _leftover_steer: - pending = _leftover_steer - logger.debug("Delivering leftover /steer as next turn: '%s...'", pending[:40]) + if result and not pending and not pending_event and result.get("pending_steer"): + pending = result.get("pending_steer") + logger.debug("Delivering leftover /steer as next turn: '%s...'", pending[:40]) - # Safety net: if the pending text is a slash command (e.g. "/stop", "/new"), discard it - # — commands should never be passed to the agent as user input. + # Safety net: a pending slash command (e.g. "/stop", "/new") is discarded — commands must + # never be passed to the agent as user input. if pending and pending.strip().startswith("/"): _pending_parts = pending.strip().split(None, 1) _pending_cmd_word = _pending_parts[0][1:].lower() if _pending_parts else "" if _pending_cmd_word: - try: + with suppress(Exception): from hermes_cli.commands import resolve_command as _rc_pending if _rc_pending(_pending_cmd_word): logger.info( "Discarding command '/%s' from pending queue — " - "commands must not be passed as agent input", - _pending_cmd_word, + "commands must not be passed as agent input", _pending_cmd_word, ) pending_event = None pending = None - except Exception: - pass if self._draining and (pending_event or pending): logger.info( "Discarding pending follow-up for session %s during gateway %s", - session_key or "?", - self._status_action_label(), + session_key or "?", self._status_action_label(), ) pending_event = None pending = None return pending_event, pending async def _run_agent_deliver_first_response( - self, - *, - source: SessionSource, - adapter: Any, - session_key: Optional[str], - run_generation: Optional[int], - event_message_id: Optional[str], - response: Any, - result: Any, - stream_consumer_holder: list, - stream_task: Any, + self, *, source: SessionSource, adapter: Any, session_key: Optional[str], + run_generation: Optional[int], event_message_id: Optional[str], response: Any, result: Any, + stream_consumer_holder: list, stream_task: Any, _status_thread_metadata: Optional[Dict[str, Any]], ) -> None: - # Queued message after normal completion: deliver the first response before the - # queued follow-up, unless streaming already delivered it. + """Deliver the first response before a queued follow-up runs, unless streaming already did.""" _sc = stream_consumer_holder[0] if _sc and stream_task: try: @@ -5051,24 +3612,19 @@ class GatewayTurnMixin: await stream_task except Exception as e: logger.debug("Stream consumer wait before queued message failed: %s", e) - # The queued branch needs raw ``result`` for interruption, history, and - # recursion state, but delivery must use the finalized task result — it carries - # empty/failure normalization and final-response processing from _run_agent_task. + # The queued branch needs raw ``result`` for interruption, history, and recursion state, but + # delivery must use the finalized task result — it carries empty/failure normalization and + # final-response processing from _run_agent_task. _delivery_result = response if isinstance(response, dict) else (result or {}) - _previewed = bool(_delivery_result.get("response_previewed")) first_response = _delivery_result.get("final_response", "") _already_streamed = self._run_agent_stream_confirmed_final_delivery( - _sc, - first_response, - previewed=_previewed, + _sc, first_response, previewed=bool(_delivery_result.get("response_previewed")), ) - # Same predicate as the normal completed-turn path: this direct queued-send branch - # predates intentional-silence filtering and would leak the literal marker. + # Same predicate as the normal completed-turn path: this direct queued-send branch predates + # intentional-silence filtering and would leak the literal marker. try: from gateway.response_filters import is_intentional_silence_agent_result - _intentional_silence = is_intentional_silence_agent_result( - _delivery_result, first_response, - ) + _intentional_silence = is_intentional_silence_agent_result(_delivery_result, first_response) except Exception: _intentional_silence = False if _intentional_silence: @@ -5078,71 +3634,39 @@ class GatewayTurnMixin: ) elif first_response: try: - if _already_streamed: - logger.info( - "Queued follow-up for session %s: final text delivery confirmed; delivering explicit media before continuing.", - session_key or "?", - ) - else: - logger.info( - "Queued follow-up for session %s: final stream delivery not confirmed; sending first response before continuing.", - session_key or "?", - ) + logger.info( + "Queued follow-up for session %s: final text delivery confirmed; delivering explicit media before continuing." + if _already_streamed else + "Queued follow-up for session %s: final stream delivery not confirmed; sending first response before continuing.", + session_key or "?", + ) await self._deliver_queued_first_response( - first_response, - source=source, - adapter=adapter, - metadata=_status_thread_metadata, - event_message_id=event_message_id, + first_response, source=source, adapter=adapter, + metadata=_status_thread_metadata, event_message_id=event_message_id, text_already_delivered=_already_streamed, - deliver_media=not _delivery_result.get("failed"), - stream_consumer=_sc, + deliver_media=not _delivery_result.get("failed"), stream_consumer=_sc, ) except Exception as e: logger.warning("Failed to send first response before queued message: %s", e) - # Release deferred bg-review notifications now that the first response is delivered: - # pop from the adapter's callback dict (no double-fire in base.py's finally) and call. + # Release deferred bg-review notifications now that the first response is delivered: pop from + # the adapter's callback dict (no double-fire in base.py's finally) and call. + _bg_cb = None if getattr(type(adapter), "pop_post_delivery_callback", None) is not None: - _bg_cb = adapter.pop_post_delivery_callback( - session_key, - generation=run_generation, - ) - if callable(_bg_cb): - try: - _bg_result = _bg_cb() - if inspect.isawaitable(_bg_result): - await _bg_result - except Exception: - pass + _bg_cb = adapter.pop_post_delivery_callback(session_key, generation=run_generation) elif adapter and hasattr(adapter, "_post_delivery_callbacks"): _bg_cb = adapter._post_delivery_callbacks.pop(session_key, None) - if callable(_bg_cb): - try: - _bg_result = _bg_cb() - if inspect.isawaitable(_bg_result): - await _bg_result - except Exception: - pass + if callable(_bg_cb): + with suppress(Exception): + _bg_result = _bg_cb() + if inspect.isawaitable(_bg_result): + await _bg_result async def _run_agent_queued_followup( - self, - *, - source: SessionSource, - adapter: Any, - session_id: str, - session_key: Optional[str], - run_generation: Optional[int], - _interrupt_depth: int, - event_message_id: Optional[str], - context_prompt: str, - history: List[Dict[str, Any]], - pending: Optional[str], - pending_event: Any, - response: Any, - result: Any, - result_holder: list, - stream_consumer_holder: list, - stream_task: Any, + self, *, source: SessionSource, adapter: Any, session_id: str, session_key: Optional[str], + run_generation: Optional[int], _interrupt_depth: int, event_message_id: Optional[str], + context_prompt: str, history: List[Dict[str, Any]], pending: Optional[str], + pending_event: Any, response: Any, result: Any, result_holder: list, + stream_consumer_holder: list, stream_task: Any, _status_thread_metadata: Optional[Dict[str, Any]], ) -> Any: """Run the queued / interrupting follow-up as the next turn (recursive ``_run_agent``).""" @@ -5154,13 +3678,12 @@ class GatewayTurnMixin: if adapter and hasattr(adapter, '_active_sessions') and session_key and session_key in adapter._active_sessions: adapter._active_sessions[session_key].clear() - # Cap recursion depth to prevent resource exhaustion when the - # user sends multiple messages while the agent keeps failing. (#816) + # Cap recursion depth: resource exhaustion when the user keeps sending messages while the + # agent keeps failing. if _interrupt_depth >= self._MAX_INTERRUPT_DEPTH: logger.warning( "Interrupt recursion depth %d reached for session %s — " - "queueing message instead of recursing.", - _interrupt_depth, session_key, + "queueing message instead of recursing.", _interrupt_depth, session_key, ) adapter = self._adapter_for_source(source) if adapter and pending_event: @@ -5169,22 +3692,15 @@ class GatewayTurnMixin: adapter.queue_message(session_key, pending) return result_holder[0] or {"final_response": response, "messages": history} - was_interrupted = result.get("interrupted") - if not was_interrupted: + # Interrupted: discard the response ("Operation interrupted." is noise; the user knows they + # sent a new message). + if not result.get("interrupted"): await self._run_agent_deliver_first_response( - source=source, - adapter=adapter, - session_key=session_key, - run_generation=run_generation, - event_message_id=event_message_id, - response=response, - result=result, - stream_consumer_holder=stream_consumer_holder, - stream_task=stream_task, - _status_thread_metadata=_status_thread_metadata, + source=source, adapter=adapter, session_key=session_key, + run_generation=run_generation, event_message_id=event_message_id, response=response, + result=result, stream_consumer_holder=stream_consumer_holder, + stream_task=stream_task, _status_thread_metadata=_status_thread_metadata, ) - # else: interrupted — discard the response ("Operation interrupted." is noise; the user - # knows they sent a new message). updated_history = result.get("messages", history) next_source = source @@ -5192,8 +3708,8 @@ class GatewayTurnMixin: next_message_id = None next_channel_prompt = None next_session_key = session_key - # Carry the pending event's message_type into the recursive call so queued voice turns - # can stream TTS and re-mark the generation for the final delivered turn. + # Carry the pending event's message_type into the recursive call so queued voice turns can + # stream TTS and re-mark the generation for the final delivered turn. next_message_type = None if pending_event is not None: next_source = getattr(pending_event, "source", None) or source @@ -5204,21 +3720,17 @@ class GatewayTurnMixin: ) return result # Resolve the follow-up's session key BEFORE preparing the inbound text: - # _prepare_inbound_message_text buffers native image paths under the key given, and - # the recursive _run_agent consumes them under next_session_key — mismatch drops them. + # _prepare_inbound_message_text buffers native image paths under the key given, and the + # recursive _run_agent consumes them under next_session_key — mismatch drops them. try: next_session_key = self._session_key_for_source(next_source) except Exception: logger.debug( "Queued follow-up session-key resolution failed; reusing %s", - session_key or "?", - exc_info=True, + session_key or "?", exc_info=True, ) next_message = await self._prepare_profile_scoped_inbound_message_text( - event=pending_event, - source=next_source, - history=updated_history, - session_key=next_session_key, + event=pending_event, source=next_source, history=updated_history, session_key=next_session_key, ) if next_message is None: return result @@ -5226,27 +3738,22 @@ class GatewayTurnMixin: next_channel_prompt = getattr(pending_event, "channel_prompt", None) next_message_type = getattr(pending_event, "message_type", None) - # Clear the prior logical turn's completed streaming marker so the recursive turn's - # streaming TTS isn't suppressed by that completion. + # Clear the prior logical turn's completed streaming marker so the recursive turn's streaming + # TTS isn't suppressed by that completion. _clear_adapter = self._adapter_for_source(source) if _clear_adapter is not None and session_key and run_generation is not None: _completed_turns = getattr(_clear_adapter, "_streaming_tts_completed_turns", None) - if _completed_turns is not None: - _prior_key = getattr(_clear_adapter, "_streaming_tts_turn_key", None) - if callable(_prior_key): - _pk = _prior_key(session_key, run_generation) - if _pk: - _completed_turns.discard(_pk) + _prior_key = getattr(_clear_adapter, "_streaming_tts_turn_key", None) + if _completed_turns is not None and callable(_prior_key): + _pk = _prior_key(session_key, run_generation) + if _pk: + _completed_turns.discard(_pk) - # Restart the typing indicator for the follow-up turn; the outer - # _process_message_background typing task is alive but may be stale. - _followup_adapter = self._adapter_for_source(source) - if _followup_adapter: + # Restart the typing indicator for the follow-up turn; the outer _process_message_background + # typing task is alive but may be stale. + if _clear_adapter: with suppress(Exception): - await _followup_adapter.send_typing( - source.chat_id, - metadata=_status_thread_metadata, - ) + await _clear_adapter.send_typing(source.chat_id, metadata=_status_thread_metadata) # Re-baseline the cached agent's message_count before recursing into the /queue follow-up: # the coherence guard would otherwise rebuild on OUR OWN flushed rows and destroy the @@ -5254,53 +3761,30 @@ class GatewayTurnMixin: await self._refresh_agent_cache_message_count(session_key, session_id) followup_result = await self._run_agent( - message=next_message, - context_prompt=context_prompt, - history=updated_history, - source=next_source, - session_id=session_id, - session_key=next_session_key, - run_generation=run_generation, - _interrupt_depth=_interrupt_depth + 1, - event_message_id=next_message_id, - channel_prompt=next_channel_prompt, + message=next_message, context_prompt=context_prompt, history=updated_history, + source=next_source, session_id=session_id, session_key=next_session_key, + run_generation=run_generation, _interrupt_depth=_interrupt_depth + 1, + event_message_id=next_message_id, channel_prompt=next_channel_prompt, message_type=next_message_type, ) return _preserve_queued_followup_history_offset(result, followup_result) async def _run_agent_cleanup_turn_tasks( - self, - *, - progress_task: Any, - log_task: Any, - interrupt_monitor: "asyncio.Task", - _notify_task: "asyncio.Task", - tracking_task: "asyncio.Task", - stream_task: Any, - stream_consumer_holder: list, - streaming_tts_consumer_holder: list, - session_key: Optional[str], - run_generation: Optional[int], + self, *, progress_task: Any, log_task: Any, interrupt_monitor: "asyncio.Task", + _notify_task: "asyncio.Task", tracking_task: "asyncio.Task", stream_task: Any, + stream_consumer_holder: list, streaming_tts_consumer_holder: list, + session_key: Optional[str], run_generation: Optional[int], ) -> None: """``finally`` half of a turn: cancel background tasks, flush stream, release the session slot.""" - # Stop progress sender, interrupt monitor, and notification task - if progress_task: - progress_task.cancel() - if log_task: - log_task.cancel() - interrupt_monitor.cancel() - _notify_task.cancel() + for task in (progress_task, log_task, interrupt_monitor, _notify_task): + if task: + task.cancel() - # Wait for stream consumer to finish its final edit if stream_task: - # If the agent never created a stream consumer (non-streaming path, or a test stub - # returning synchronously) there is nothing to flush — cancel now instead of waiting - # out the 5s timeout polling for a consumer that will never arrive. - _has_stream_consumer = ( - stream_consumer_holder - and stream_consumer_holder[0] is not None - ) - if not _has_stream_consumer: + # If the agent never created a stream consumer (non-streaming path, or a test stub returning + # synchronously) there is nothing to flush — cancel now instead of waiting out the 5s + # timeout polling for a consumer that will never arrive. + if not (stream_consumer_holder and stream_consumer_holder[0] is not None): stream_task.cancel() with suppress(asyncio.CancelledError): await stream_task @@ -5320,181 +3804,150 @@ class GatewayTurnMixin: with suppress(Exception): await _stts_finally.wait_complete(timeout=2.0) - # Clean up tracking tracking_task.cancel() if session_key: - # Release the slot only if this run's generation still owns it: a /stop or /new that - # bumped the generation while we unwound already installed its own state; keep it. - self._release_running_agent_state( - session_key, run_generation=run_generation - ) + # Release the slot only if this run's generation still owns it: a /stop or /new that bumped + # the generation while we unwound already installed its own state; keep it. + self._release_running_agent_state(session_key, run_generation=run_generation) if self._draining: self._update_runtime_status("draining") - # Wait for cancelled tasks - for task in [progress_task, log_task, interrupt_monitor, tracking_task, _notify_task]: + for task in (progress_task, log_task, interrupt_monitor, tracking_task, _notify_task): if task: try: await task except asyncio.CancelledError: pass except Exception: - # A background task that died of a non-cancellation error (transport drop in - # a progress/card publish) must not abort the cleanup path — everything - # after this loop (final-delivery bookkeeping) still runs (review B7). - logger.debug( - "background turn task failed during cleanup", - exc_info=True, - ) + # A background task that died of a non-cancellation error (transport drop in a + # progress/card publish) must not abort the cleanup path — final-delivery + # bookkeeping after this loop still runs. + logger.debug("background turn task failed during cleanup", exc_info=True) async def _run_agent_mark_streamed_delivery( - self, - response: Any, - stream_consumer_holder: list, - source: SessionSource, + self, response: Any, stream_consumer_holder: list, source: SessionSource, session_key: Optional[str], ) -> None: - # If streaming already delivered the response, skip the caller's send() — but never when the - # agent failed (the error is unseen content) or on "(empty)": interim text ("Let me search…") - # set already_sent but is NOT the final answer; suppressing would leave the user with silence. + """Set ``response["already_sent"]`` when streaming already delivered the final reply. + + Never when the agent failed (the error is unseen content) or on "(empty)": interim text + ("Let me search…") set already_sent but is NOT the final answer; suppressing would leave the + user with silence. Both suppression flags reflect call success, not content, so reconcile + against the recorded turn-final payload: on mismatch (False, incl. payload-less split + delivery) neither flag may suppress the final send; None (no record) keeps legacy trust. + """ _sc = stream_consumer_holder[0] - if isinstance(response, dict) and not response.get("failed"): - _final = response.get("final_response") or "" - _is_empty_sentinel = not _final or _final == "(empty)" - # response_previewed means interim_assistant_callback already saw the final text, but only - # suppress the send if that exact text was delivered — unrelated commentary/progress isn't it. - _previewed = bool(response.get("response_previewed")) - _content_delivered = bool( - _sc and getattr(_sc, "final_content_delivered", False) + if not isinstance(response, dict) or response.get("failed"): + return + _final = response.get("final_response") or "" + _is_empty_sentinel = not _final or _final == "(empty)" + # response_previewed means interim_assistant_callback already saw the final text, but only + # suppress the send if that exact text was delivered — unrelated commentary/progress isn't it. + _previewed = bool(response.get("response_previewed")) + _content_delivered = bool(_sc and getattr(_sc, "final_content_delivered", False)) + _stale_finalized = False + if _content_delivered and not _is_empty_sentinel: + _matcher = getattr(_sc, "delivered_final_matches", None) + if callable(_matcher): + try: + _stale_finalized = _matcher(_final) is False + except Exception: + _stale_finalized = False + if _stale_finalized: + _content_delivered = False + # Plugin hooks (e.g. transform_llm_output) may append content after streaming finished — when + # transformed, always send the final version so the appended content reaches the client. + _transformed = bool(response.get("response_transformed")) + # Suppress the normal send only when the actual final reply reached the user (streamed, or + # interim preview of that *exact* text); commentary shown during a compression/split isn't it. + _streamed = self._run_agent_stream_confirmed_final_delivery(_sc, _final, previewed=_previewed) + if _is_empty_sentinel: + return + if not _transformed and (_streamed or _content_delivered): + logger.info( + "Suppressing normal final send for session %s: final delivery already confirmed (streamed=%s previewed=%s content_delivered=%s).", + session_key or "?", _streamed, _previewed, _content_delivered, ) - # A *successful* finalize edit can still carry only the last preview snapshot, and both - # suppression flags reflect call success, not content. Reconcile against the recorded - # turn-final payload: on mismatch (False, incl. payload-less split delivery) neither flag - # may suppress the final send; None (no record) keeps legacy trust. - _stale_finalized = False - if _content_delivered and not _is_empty_sentinel: - _matcher = getattr(_sc, "delivered_final_matches", None) - if callable(_matcher): - try: - _stale_finalized = _matcher(_final) is False - except Exception: - _stale_finalized = False - if _stale_finalized: - _content_delivered = False - # Plugin hooks (e.g. transform_llm_output) may append content after streaming finished — when - # transformed, always send the final version so the appended content reaches the client. - _transformed = bool(response.get("response_transformed")) - # Suppress the normal send only when the actual final reply reached the user (streamed, or - # interim preview of that *exact* text); commentary shown during a compression/split isn't it. - _streamed = self._run_agent_stream_confirmed_final_delivery( - _sc, - _final, - previewed=_previewed, - ) - if not _is_empty_sentinel and not _transformed and (_streamed or _content_delivered): + response["already_sent"] = True + elif not _transformed and _stale_finalized and _sc is not None: + # Stale finalize: the streamed message holds only the last preview snapshot. Edit it up to + # the complete response; on edit failure leave already_sent unset so the normal send + # delivers. Not for split delivery: message_id is only the LAST chunk, so editing it would + # repeat every sealed head chunk — fall through to the normal send. + _sc_msg_id = _sc.message_id + _sc_adapter = getattr(_sc, "adapter", None) + if getattr(_sc, "_turn_split_delivery", False): logger.info( - "Suppressing normal final send for session %s: final delivery already confirmed (streamed=%s previewed=%s content_delivered=%s).", + "Stale streamed finalize detected for session %s on a multi-message split; skipping the in-place reconciliation edit and delivering the complete response via normal final send (#78541).", session_key or "?", - _streamed, - _previewed, - _content_delivered, ) - response["already_sent"] = True - elif not _is_empty_sentinel and not _transformed and _stale_finalized and _sc is not None: - # Stale finalize: the streamed message holds only the last preview snapshot. Edit it - # up to the complete response; on edit failure leave already_sent unset so the normal - # send delivers. Not for split delivery: message_id is only the LAST chunk, so editing - # it would repeat every sealed head chunk — fall through to the normal send. - _sc_msg_id = _sc.message_id - _sc_adapter = getattr(_sc, "adapter", None) - if getattr(_sc, "_turn_split_delivery", False): - logger.info( - "Stale streamed finalize detected for session %s on a multi-message split; skipping the in-place reconciliation edit and delivering the complete response via normal final send (#78541).", - session_key or "?", + elif _sc_msg_id and _sc_msg_id != "__no_edit__" and _sc_adapter is not None: + try: + _reconcile_res = await _sc_adapter.edit_message( + chat_id=source.chat_id, message_id=_sc_msg_id, content=_final, finalize=True, ) - elif _sc_msg_id and _sc_msg_id != "__no_edit__" and _sc_adapter is not None: - try: - _reconcile_res = await _sc_adapter.edit_message( - chat_id=source.chat_id, - message_id=_sc_msg_id, - content=_final, - finalize=True, - ) - if getattr(_reconcile_res, "success", True): - response["already_sent"] = True - logger.info( - "Reconciled stale streamed finalize for session %s: edited message %s with the complete response (#71643).", - session_key or "?", _sc_msg_id, - ) - else: - logger.warning( - "Stale-finalize reconciliation edit failed for session %s (%s); sending complete response via normal final send.", - session_key or "?", - getattr(_reconcile_res, "error", None), - ) - except Exception as _edit_err: - logger.warning( - "Stale-finalize reconciliation edit failed for session %s: %s; sending complete response via normal final send.", - session_key or "?", _edit_err, - ) - else: - logger.info( - "Stale streamed finalize detected for session %s with no editable message; delivering complete response via normal final send (#71643).", - session_key or "?", - ) - elif not _is_empty_sentinel and _transformed and _sc is not None: - # Plugin hooks transformed the response after streaming — edit the - # existing streamed message instead of sending a duplicate. - _sc_msg_id = _sc.message_id - if _sc_msg_id: - try: - await _sc.adapter.edit_message( - chat_id=source.chat_id, - message_id=_sc_msg_id, - content=response["final_response"], - finalize=True, - ) + if getattr(_reconcile_res, "success", True): response["already_sent"] = True logger.info( - "Edited streamed message %s for session %s to include plugin-transformed content.", - _sc_msg_id, session_key or "?", + "Reconciled stale streamed finalize for session %s: edited message %s with the complete response (#71643).", + session_key or "?", _sc_msg_id, ) - except Exception as _edit_err: + else: logger.warning( - "Failed to edit streamed message for session %s: %s", - session_key or "?", _edit_err, + "Stale-finalize reconciliation edit failed for session %s (%s); sending complete response via normal final send.", + session_key or "?", getattr(_reconcile_res, "error", None), ) - elif _sc is not None and not _is_empty_sentinel: - # DUPLICATE-RISK DIAGNOSTIC: a stream consumer existed for this turn but suppression - # did NOT fire, so the gateway's normal final-send is about to run. Log the decision - # inputs so a recurrence can be pinned to "signal never set" vs "ack-pending race". - logger.warning( - "Normal final-send NOT suppressed despite active stream " - "consumer for session %s: streamed=%s previewed=%s " - "content_delivered=%s transformed=%s final_len=%d — " - "possible duplicate send (see wecom ack-timeout RCA).", + except Exception as _edit_err: + logger.warning( + "Stale-finalize reconciliation edit failed for session %s: %s; sending complete response via normal final send.", + session_key or "?", _edit_err, + ) + else: + logger.info( + "Stale streamed finalize detected for session %s with no editable message; delivering complete response via normal final send (#71643).", session_key or "?", - _streamed, - _previewed, - _content_delivered, - _transformed, - len(_final), ) + elif _transformed and _sc is not None: + # Plugin hooks transformed the response after streaming — edit the existing streamed + # message instead of sending a duplicate. + _sc_msg_id = _sc.message_id + if _sc_msg_id: + try: + await _sc.adapter.edit_message( + chat_id=source.chat_id, message_id=_sc_msg_id, content=response["final_response"], finalize=True, + ) + response["already_sent"] = True + logger.info( + "Edited streamed message %s for session %s to include plugin-transformed content.", + _sc_msg_id, session_key or "?", + ) + except Exception as _edit_err: + logger.warning( + "Failed to edit streamed message for session %s: %s", session_key or "?", _edit_err, + ) + elif _sc is not None: + # DUPLICATE-RISK DIAGNOSTIC: a stream consumer existed but suppression did NOT fire, so the + # normal final-send is about to run. Log the decision inputs so a recurrence can be pinned + # to "signal never set" vs "ack-pending race". + logger.warning( + "Normal final-send NOT suppressed despite active stream " + "consumer for session %s: streamed=%s previewed=%s " + "content_delivered=%s transformed=%s final_len=%d — " + "possible duplicate send (see wecom ack-timeout RCA).", + session_key or "?", _streamed, _previewed, _content_delivered, _transformed, len(_final), + ) def _run_agent_schedule_bubble_cleanup( - self, - response: Any, - _cleanup_progress: bool, - _cleanup_adapter: Any, - _cleanup_msg_ids: List[str], - source: SessionSource, - session_key: Optional[str], + self, response: Any, _cleanup_progress: bool, _cleanup_adapter: Any, + _cleanup_msg_ids: List[str], source: SessionSource, session_key: Optional[str], run_generation: Optional[int], ) -> None: - # Schedule deletion of tracked temporary progress bubbles after the final response lands; failed - # runs keep them as breadcrumbs. Only on adapters with ``delete_message``; failures swallowed. + """Schedule deletion of tracked temporary progress bubbles after the final response lands. + + Failed runs keep them as breadcrumbs. Only on adapters with ``delete_message``; failures swallowed. + """ from gateway.run import safe_schedule_threadsafe - if ( + if not ( _cleanup_progress and _cleanup_adapter is not None and _cleanup_msg_ids @@ -5503,116 +3956,75 @@ class GatewayTurnMixin: and not response.get("failed") and hasattr(_cleanup_adapter, "register_post_delivery_callback") ): - _ids_snapshot = list(_cleanup_msg_ids) - _chat_id_snapshot = source.chat_id - _adapter_snapshot = _cleanup_adapter - _loop_snapshot = asyncio.get_running_loop() + return + _ids_snapshot = list(_cleanup_msg_ids) + _chat_id_snapshot = source.chat_id + _loop_snapshot = asyncio.get_running_loop() - def _cleanup_temp_bubbles() -> None: - async def _delete_all() -> None: - for _mid in _ids_snapshot: - with suppress(Exception): - await _adapter_snapshot.delete_message( - _chat_id_snapshot, _mid - ) - with suppress(Exception): - safe_schedule_threadsafe( - _delete_all(), _loop_snapshot, - logger=logger, - log_message="Temp bubble cleanup scheduling error", - ) - - try: - _cleanup_adapter.register_post_delivery_callback( - session_key, - _cleanup_temp_bubbles, - generation=run_generation, + def _cleanup_temp_bubbles() -> None: + async def _delete_all() -> None: + for _mid in _ids_snapshot: + with suppress(Exception): + await _cleanup_adapter.delete_message(_chat_id_snapshot, _mid) + with suppress(Exception): + safe_schedule_threadsafe( + _delete_all(), _loop_snapshot, logger=logger, + log_message="Temp bubble cleanup scheduling error", ) - except Exception as _rpe: - logger.debug("Post-delivery cleanup registration failed: %s", _rpe) + + try: + _cleanup_adapter.register_post_delivery_callback( + session_key, _cleanup_temp_bubbles, generation=run_generation, + ) + except Exception as _rpe: + logger.debug("Post-delivery cleanup registration failed: %s", _rpe) def _run_agent_bind_turn_wiring( - self, - turn_ctx: TurnContext, - turn_runner: TurnRunner, - source: SessionSource, - event_message_id: Optional[str], - _progress_metadata: Optional[dict], - _progress_reply_to: Optional[str], - _progress_thread_id: Any, + self, turn_ctx: TurnContext, turn_runner: TurnRunner, source: SessionSource, + event_message_id: Optional[str], _progress_metadata: Optional[dict], + _progress_reply_to: Optional[str], _progress_thread_id: Any, _relay_prospective_thread_id: Optional[str], ) -> Optional[Dict[str, Any]]: """Publish progress metadata, result holders and the sync→async bridges onto ``turn_ctx``. Returns ``_status_thread_metadata``; the holders are read back via ``turn_ctx.*_holder``. + The one-slot holders share agent / result / tool-defs / stream consumer / streaming-TTS + consumer between run_sync's executor thread and the outer finalisation + interrupt paths. """ - # Extracted to TurnRunner.send_progress_messages; the threading metadata above is published - # onto the shared TurnContext where the original closure's captured locals were bound. turn_ctx._progress_metadata = _progress_metadata turn_ctx._progress_reply_to = _progress_reply_to - - # We need to share the agent instance for interrupt support - agent_holder = [None] # Mutable container for the agent instance - turn_ctx.agent_holder = agent_holder - result_holder = [None] # Mutable container for the result - tools_holder = [None] # Mutable container for the tool definitions - stream_consumer_holder = [None] # Mutable container for stream consumer - # streaming PCM audio consumer. Created on the gateway event-loop thread (NOT in run_sync's - # executor worker) so outer finalisation / interrupt paths can reference it without a NameError. - streaming_tts_consumer_holder: list = [None] - turn_ctx.result_holder = result_holder - turn_ctx.tools_holder = tools_holder - turn_ctx.stream_consumer_holder = stream_consumer_holder - turn_ctx.streaming_tts_consumer_holder = streaming_tts_consumer_holder - - # Bridge sync step_callback → async hooks.emit for agent:step events - _loop_for_step = asyncio.get_running_loop() - _hooks_ref = self.hooks - - # Bridge extracted to TurnRunner._step_callback_sync; the loop and - # hooks refs bound just above are published at their original site. - turn_ctx._loop_for_step = _loop_for_step - turn_ctx._hooks_ref = _hooks_ref + turn_ctx.agent_holder = [None] + turn_ctx.result_holder = [None] + turn_ctx.tools_holder = [None] + turn_ctx.stream_consumer_holder = [None] + turn_ctx.streaming_tts_consumer_holder = [None] + # Bridges: sync step_callback / event_callback / status_callback → async hooks.emit and + # adapter.send (TurnRunner methods bound onto the shared ctx). + turn_ctx._loop_for_step = asyncio.get_running_loop() + turn_ctx._hooks_ref = self.hooks turn_ctx._step_callback_sync = turn_runner._step_callback_sync - - # Bridge sync event_callback → async hooks.emit for lifecycle events (e.g. session:compress - # after a compression split); extracted to TurnRunner._event_callback_sync. turn_ctx._event_callback_sync = turn_runner._event_callback_sync - - # Bridge sync status_callback → async adapter.send for context pressure - _status_adapter = self._adapter_for_source(source) - _status_chat_id = source.chat_id + turn_ctx._status_adapter = self._adapter_for_source(source) _status_thread_metadata = self._run_agent_status_thread_metadata( source, event_message_id, _progress_thread_id, _relay_prospective_thread_id, ) - - # Bridge extracted to TurnRunner._status_callback_sync; publish the status wiring computed - # above onto the shared TurnContext at the exact original binding site. - turn_ctx._status_adapter = _status_adapter - turn_ctx._status_chat_id = _status_chat_id + turn_ctx._status_chat_id = source.chat_id turn_ctx._status_thread_metadata = _status_thread_metadata turn_ctx._status_callback_sync = turn_runner._status_callback_sync return _status_thread_metadata async def _run_agent_notify_long_running( - self, - disp: "GatewayRunner._RunAgentDisplay", - *, - source: SessionSource, - session_key: Optional[str], - agent_holder: list, - _executor_task_holder: list, - _NOTIFY_INTERVAL: Optional[float], - _long_running_mode: str, - _notify_start: float, - _status_thread_metadata: Optional[Dict[str, Any]], - _cleanup_progress: bool, + self, disp: "GatewayRunner._RunAgentDisplay", *, source: SessionSource, + session_key: Optional[str], agent_holder: list, _executor_task_holder: list, + _NOTIFY_INTERVAL: Optional[float], _long_running_mode: str, _notify_start: float, + _status_thread_metadata: Optional[Dict[str, Any]], _cleanup_progress: bool, _cleanup_msg_ids: List[str], ) -> None: - """Periodic \"still working\" heartbeat (edited in place where the adapter supports it). + """Periodic "still working" heartbeat, edited in place where the adapter supports it. - ``_executor_task_holder[0]`` is populated once the executor future exists; tolerate the - brief window before then (it reads as None). + Stops once this run no longer owns the session slot or the executor has finished, else a + stale "running: delegate_task" bubble outlives its run. ``_executor_task_holder[0]`` is bound + just after this task is scheduled; tolerate the brief window before then (reads as None). """ from gateway.run import _interim_metadata, _non_conversational_metadata if _NOTIFY_INTERVAL is None: @@ -5620,40 +4032,27 @@ class GatewayTurnMixin: _notify_adapter = self._adapter_for_source(source) if not _notify_adapter: return - # Track the heartbeat message id to edit in place where supported (Telegram, Discord, - # Slack, ...) instead of a new "Still working" bubble every interval. _heartbeat_msg_id: Optional[str] = None while True: await asyncio.sleep(_NOTIFY_INTERVAL) - # Stop heartbeating once this run no longer owns the session slot or the executor has - # finished, else a stale "running: delegate_task" bubble outlives its run. _executor_task - # is bound just after this task is scheduled; tolerate the brief window before then. - _exec_ref = _executor_task_holder[0] if not self._should_emit_long_running_notification( - session_key, agent_holder[0], _exec_ref + session_key, agent_holder[0], _executor_task_holder[0] ): break _elapsed_mins = int((time.time() - _notify_start) // 60) # Default heartbeat is terse (elapsed + current tool); the verbose iteration counter is # gated on busy_ack_detail so users can opt in per platform. - _agent_ref = agent_holder[0] _status_detail = "" _want_iteration_detail = bool( - disp.resolve_display_setting( - disp.user_config, - disp.platform_key, - "busy_ack_detail", - True, - ) + disp.resolve_display_setting(disp.user_config, disp.platform_key, "busy_ack_detail", True) ) + _agent_ref = agent_holder[0] if _agent_ref and hasattr(_agent_ref, "get_activity_summary"): try: _a = _agent_ref.get_activity_summary() _parts = [] if _want_iteration_detail: - _parts.append( - f"iteration {_a['api_call_count']}/{_a['max_iterations']}" - ) + _parts.append(f"iteration {_a['api_call_count']}/{_a['max_iterations']}") _action = _a.get("current_tool") or _a.get("last_activity_desc") if _action: _parts.append(str(_action)) @@ -5670,23 +4069,16 @@ class GatewayTurnMixin: _notify_res = None if _heartbeat_msg_id: try: - _notify_res = await _notify_adapter.edit_message( - source.chat_id, - _heartbeat_msg_id, - _heartbeat_text, - ) + _notify_res = await _notify_adapter.edit_message(source.chat_id, _heartbeat_msg_id, _heartbeat_text) except Exception as _ee: logger.debug("Heartbeat edit failed: %s", _ee) _notify_res = None if not (_notify_res and getattr(_notify_res, "success", False)): _notify_res = await _notify_adapter.send( - source.chat_id, - _heartbeat_text, + source.chat_id, _heartbeat_text, metadata=_interim_metadata(_non_conversational_metadata(_status_thread_metadata, platform=source.platform)), ) - if getattr(_notify_res, "success", False) and getattr( - _notify_res, "message_id", None - ): + if getattr(_notify_res, "success", False) and getattr(_notify_res, "message_id", None): _heartbeat_msg_id = str(_notify_res.message_id) if _cleanup_progress: _cleanup_msg_ids.append(_heartbeat_msg_id) @@ -5694,93 +4086,51 @@ class GatewayTurnMixin: logger.debug("Long-running notification error: %s", _ne) async def _run_agent_inner( - self, - message: str, - context_prompt: str, - history: List[Dict[str, Any]], - source: SessionSource, - session_id: str, - session_key: str = None, - run_generation: Optional[int] = None, - _interrupt_depth: int = 0, - event_message_id: Optional[str] = None, - inbound_message_id: Optional[str] = None, - channel_prompt: Optional[str] = None, - moa_config: Optional[dict] = None, - persist_user_message: Optional[Any] = None, - persist_user_timestamp: Optional[float] = None, - persist_user_display_kind: Optional[str] = None, - message_type: Optional[str] = None, + self, message: str, context_prompt: str, history: List[Dict[str, Any]], + source: SessionSource, session_id: str, session_key: str = None, + run_generation: Optional[int] = None, _interrupt_depth: int = 0, + event_message_id: Optional[str] = None, inbound_message_id: Optional[str] = None, + channel_prompt: Optional[str] = None, moa_config: Optional[dict] = None, + persist_user_message: Optional[Any] = None, persist_user_timestamp: Optional[float] = None, + persist_user_display_kind: Optional[str] = None, message_type: Optional[str] = None, ) -> Dict[str, Any]: """Run the agent; returns the full run_conversation result dict. Keys: "final_response", "messages", "api_calls", "completed". """ from gateway.run import _float_env - # ---- Proxy mode: delegate to remote API server ---- if self._get_proxy_url(): return await self._run_agent_via_proxy( - message=message, - context_prompt=context_prompt, - history=history, - source=source, - session_id=session_id, - session_key=session_key, - run_generation=run_generation, + message=message, context_prompt=context_prompt, history=history, source=source, + session_id=session_id, session_key=session_key, run_generation=run_generation, event_message_id=event_message_id, ) from run_agent import AIAgent disp = self._run_agent_display_settings(source) - _display_surface_mode = disp._display_surface_mode - needs_progress_queue = disp.needs_progress_queue - log_mode_enabled = disp.log_mode_enabled - log_queue = disp.log_queue - turn_ctx, turn_runner, _cleanup_adapter = self._run_agent_build_turn_context( - disp, - AIAgent, - message=message, - context_prompt=context_prompt, - history=history, - source=source, - session_id=session_id, - session_key=session_key, - run_generation=run_generation, - _interrupt_depth=_interrupt_depth, - event_message_id=event_message_id, - inbound_message_id=inbound_message_id, - channel_prompt=channel_prompt, - moa_config=moa_config, + disp, AIAgent, message=message, context_prompt=context_prompt, history=history, + source=source, session_id=session_id, session_key=session_key, + run_generation=run_generation, _interrupt_depth=_interrupt_depth, + event_message_id=event_message_id, inbound_message_id=inbound_message_id, + channel_prompt=channel_prompt, moa_config=moa_config, persist_user_message=persist_user_message, persist_user_timestamp=persist_user_timestamp, persist_user_display_kind=persist_user_display_kind, ) _cleanup_progress = turn_ctx._cleanup_progress _cleanup_msg_ids = turn_ctx._cleanup_msg_ids - ( - _progress_metadata, - _progress_reply_to, - _progress_thread_id, + _progress_metadata, _progress_reply_to, _progress_thread_id, _relay_prospective_thread_id, ) = self._run_agent_progress_threading(source, event_message_id, disp._native_slack_task_cards) - _status_thread_metadata = self._run_agent_bind_turn_wiring( - turn_ctx, - turn_runner, - source, - event_message_id, - _progress_metadata, - _progress_reply_to, - _progress_thread_id, - _relay_prospective_thread_id, + turn_ctx, turn_runner, source, event_message_id, + _progress_metadata, _progress_reply_to, _progress_thread_id, _relay_prospective_thread_id, ) - send_progress_messages = turn_runner.send_progress_messages agent_holder = turn_ctx.agent_holder result_holder = turn_ctx.result_holder - tools_holder = turn_ctx.tools_holder stream_consumer_holder = turn_ctx.stream_consumer_holder streaming_tts_consumer_holder = turn_ctx.streaming_tts_consumer_holder @@ -5788,41 +4138,20 @@ class GatewayTurnMixin: source, message_type, _status_thread_metadata, streaming_tts_consumer_holder, ) - # run_sync extracted to TurnRunner.run_sync (bound method; executor call unchanged). Its - # closed-over locals travel on turn_ctx; `nonlocal message` rebinds became ctx.message writes. - run_sync = turn_runner.run_sync - - # Start the progress sender if enabled. Gate on needs_progress_queue (tool_progress OR - # thinking_progress), not tool_progress alone: the sender drains BOTH tool-progress lines and - # _thinking scratch bubbles — a tool_progress-only gate left thinking-only queues never drained. - progress_task = None - if needs_progress_queue: - progress_task = asyncio.create_task(send_progress_messages()) - - # Start the tool-call log writer when tool_progress == "log". - log_task = None - if log_mode_enabled: - log_task = asyncio.create_task(self._run_agent_write_tool_log(log_queue)) - - # Start stream consumer task — polls for consumer creation since it - # happens inside run_sync (thread pool) after the agent is constructed. - stream_task = None + # Progress sender gates on needs_progress_queue (tool_progress OR thinking_progress), not + # tool_progress alone: it drains BOTH tool-progress lines and _thinking scratch bubbles. + progress_task = asyncio.create_task(turn_runner.send_progress_messages()) if disp.needs_progress_queue else None + log_task = asyncio.create_task(self._run_agent_write_tool_log(disp.log_queue)) if disp.log_mode_enabled else None + # The stream consumer is created inside run_sync (thread pool) after the agent is constructed; + # this task polls for it. stream_task = asyncio.create_task(self._run_agent_stream_consumer_task(stream_consumer_holder)) - - # Track this agent as running for this session (for interrupt support) - # We do this in a callback after the agent is created tracking_task = asyncio.create_task( self._run_agent_track_agent(session_key, run_generation, agent_holder) ) - _interrupt_detected = asyncio.Event() # shared with backup check interrupt_monitor = asyncio.create_task( self._run_agent_monitor_for_interrupt( - source, - session_key, - agent_holder, - _interrupt_detected, - streaming_tts_consumer_holder, + source, session_key, agent_holder, _interrupt_detected, streaming_tts_consumer_holder, ) ) @@ -5830,108 +4159,63 @@ class GatewayTurnMixin: # agent.gateway_notify_interval or HERMES_AGENT_NOTIFY_INTERVAL env; default 180s. _NOTIFY_INTERVAL_RAW = _float_env("HERMES_AGENT_NOTIFY_INTERVAL", 180) _NOTIFY_INTERVAL = _NOTIFY_INTERVAL_RAW if _NOTIFY_INTERVAL_RAW > 0 else None - _long_running_mode = _display_surface_mode( - "long_running_notifications", - default=True, - allow_generic=True, - ) + _long_running_mode = disp._display_surface_mode("long_running_notifications", default=True, allow_generic=True) if _long_running_mode == "off": _NOTIFY_INTERVAL = None - _notify_start = time.time() _executor_task_holder: list = [None] # bound once the executor future exists (see below) _notify_task = asyncio.create_task( self._run_agent_notify_long_running( - disp, - source=source, - session_key=session_key, - agent_holder=agent_holder, - _executor_task_holder=_executor_task_holder, - _NOTIFY_INTERVAL=_NOTIFY_INTERVAL, - _long_running_mode=_long_running_mode, - _notify_start=_notify_start, + disp, source=source, session_key=session_key, agent_holder=agent_holder, + _executor_task_holder=_executor_task_holder, _NOTIFY_INTERVAL=_NOTIFY_INTERVAL, + _long_running_mode=_long_running_mode, _notify_start=time.time(), _status_thread_metadata=_status_thread_metadata, - _cleanup_progress=_cleanup_progress, - _cleanup_msg_ids=_cleanup_msg_ids, + _cleanup_progress=_cleanup_progress, _cleanup_msg_ids=_cleanup_msg_ids, ) ) try: + # run_sync is TurnRunner.run_sync (bound method; executor call unchanged). worker = self._run_agent_start_turn_worker( - turn_ctx, run_sync, agent_holder, session_id, session_key, run_generation, + turn_ctx, turn_runner.run_sync, agent_holder, session_id, session_key, run_generation, ) _executor_task_holder[0] = worker.executor_task # read late by _notify_long_running response = await self._run_agent_await_turn_worker( - worker, - source=source, - session_key=session_key, - agent_holder=agent_holder, - result_holder=result_holder, - tools_holder=tools_holder, - _interrupt_detected=_interrupt_detected, - interrupt_monitor=interrupt_monitor, + worker, source=source, session_key=session_key, agent_holder=agent_holder, + result_holder=result_holder, tools_holder=turn_ctx.tools_holder, + _interrupt_detected=_interrupt_detected, interrupt_monitor=interrupt_monitor, streaming_tts_consumer_holder=streaming_tts_consumer_holder, _status_thread_metadata=_status_thread_metadata, ) - self._run_agent_evict_on_fallback(session_key, agent_holder, result_holder) - # Check if we were interrupted OR have a queued message (/queue). + # Interrupted OR queued message (/queue)? result = result_holder[0] adapter = self._adapter_for_source(source) - await self._run_agent_finalize_streaming_tts( streaming_tts_consumer_holder, adapter, session_key, run_generation, ) - - pending_event, pending = await self._run_agent_drain_pending( - result, adapter, source, session_key, - ) - + pending_event, pending = await self._run_agent_drain_pending(result, adapter, source, session_key) if pending_event or pending: return await self._run_agent_queued_followup( - source=source, - adapter=adapter, - session_id=session_id, - session_key=session_key, - run_generation=run_generation, - _interrupt_depth=_interrupt_depth, - event_message_id=event_message_id, - context_prompt=context_prompt, - history=history, - pending=pending, - pending_event=pending_event, - response=response, - result=result, - result_holder=result_holder, - stream_consumer_holder=stream_consumer_holder, - stream_task=stream_task, + source=source, adapter=adapter, session_id=session_id, session_key=session_key, + run_generation=run_generation, _interrupt_depth=_interrupt_depth, + event_message_id=event_message_id, context_prompt=context_prompt, + history=history, pending=pending, pending_event=pending_event, + response=response, result=result, result_holder=result_holder, + stream_consumer_holder=stream_consumer_holder, stream_task=stream_task, _status_thread_metadata=_status_thread_metadata, ) finally: await self._run_agent_cleanup_turn_tasks( - progress_task=progress_task, - log_task=log_task, - interrupt_monitor=interrupt_monitor, - _notify_task=_notify_task, - tracking_task=tracking_task, - stream_task=stream_task, + progress_task=progress_task, log_task=log_task, interrupt_monitor=interrupt_monitor, + _notify_task=_notify_task, tracking_task=tracking_task, stream_task=stream_task, stream_consumer_holder=stream_consumer_holder, streaming_tts_consumer_holder=streaming_tts_consumer_holder, - session_key=session_key, - run_generation=run_generation, + session_key=session_key, run_generation=run_generation, ) - await self._run_agent_mark_streamed_delivery( - response, stream_consumer_holder, source, session_key, - ) + await self._run_agent_mark_streamed_delivery(response, stream_consumer_holder, source, session_key) self._run_agent_schedule_bubble_cleanup( - response, - _cleanup_progress, - _cleanup_adapter, - _cleanup_msg_ids, - source, - session_key, - run_generation, + response, _cleanup_progress, _cleanup_adapter, _cleanup_msg_ids, source, session_key, run_generation, ) - return response diff --git a/gateway/run_turn_runner.py b/gateway/run_turn_runner.py index edea61702c..0081d693a8 100644 --- a/gateway/run_turn_runner.py +++ b/gateway/run_turn_runner.py @@ -1,31 +1,31 @@ """Per-turn callback runner (progress/status/voice/run_sync) for the gateway agent turn. -Split out of ``gateway/run.py``; ``TurnRunner`` owns the per-turn callbacks/closures ``GatewayRunner._run_agent_inner`` binds. -``gateway.run`` internals are imported lazily inside method bodies (import cycle), -so ``patch("gateway.run.X")`` keeps intercepting them at call time. +Split out of ``gateway/run.py``; ``TurnRunner`` owns the per-turn callbacks/closures +``GatewayRunner._run_agent_inner`` binds. ``gateway.run`` internals are imported lazily inside +method bodies (import cycle), so ``patch("gateway.run.X")`` keeps intercepting them at call time. """ from __future__ import annotations -import logging -from typing import TYPE_CHECKING import asyncio import dataclasses import inspect import json +import logging import queue import re import threading import time -from agent.replay_cleanup import strip_stale_dangerous_confirmations from contextlib import suppress from datetime import datetime +from typing import TYPE_CHECKING, Any, Dict, List, Optional + +from agent.replay_cleanup import strip_stale_dangerous_confirmations from gateway.config import Platform from gateway.media_repair import repair_explicit_computer_use_media_paths from gateway.platforms.base import BasePlatformAdapter from gateway.turn_context import TurnContext from hermes_cli.config import cfg_get -from typing import Any, Dict, List, Optional from utils import is_truthy_value if TYPE_CHECKING: # string annotations only; never imported at runtime (cycle) @@ -45,58 +45,64 @@ class TurnRunner: self._runner = runner self._ctx = ctx + # ── shared thread→loop plumbing ───────────────────────────────────────────────────────── + + def _schedule(self, coro, log_message: str, loop=None): + """Hop a coroutine from the agent's sync worker thread onto the gateway loop.""" + from gateway.run import safe_schedule_threadsafe + return safe_schedule_threadsafe( + coro, self._ctx._loop_for_step if loop is None else loop, logger=logger, + log_message=log_message, + ) + + def _agent_interrupted(self) -> bool: + """True once the user sent `stop` (agent_holder[0] is the shared agent handle).""" + try: + agent = self._ctx.agent_holder[0] if self._ctx.agent_holder else None + return bool(agent is not None and getattr(agent, "is_interrupted", False)) + except Exception: + return False + + def _stream_consumer(self): + holder = self._ctx.stream_consumer_holder + return holder[0] if holder else None + + def _drain_progress_queue(self) -> None: + q = self._ctx.progress_queue + while not q.empty(): + try: + q.get_nowait() + except Exception: + break + + def _track_progress_result(self, result) -> None: + """Remember a delivered progress/status message id for end-of-turn cleanup.""" + ctx = self._ctx + if ( + ctx._cleanup_progress + and getattr(result, "success", False) + and getattr(result, "message_id", None) + ): + ctx._cleanup_msg_ids.append(str(result.message_id)) + + def _track_future_cleanup_id(self, fut) -> None: + try: + res = fut.result() + except Exception: + return + self._track_progress_result(res) + + # ── progress_callback (agent thread → progress queue) ─────────────────────────────────── + def progress_callback(self, event_type: str, tool_name: str = None, preview: str = None, args: dict = None, **kwargs): """Callback invoked by agent on tool lifecycle events.""" - from gateway.run import _hermes_home, _load_gateway_config, safe_schedule_threadsafe ctx = self._ctx # Failed subagent → one clean user-facing notice, handled FIRST, before every progress-queue - # gate: platforms with tool_progress off must still hear about a dead delegation. Only - # terminal failure statuses render (same notice rail as credit warnings); success/interrupt - # stay quiet. + # gate: platforms with tool_progress off must still hear about a dead delegation. if event_type == "subagent.complete": - _sub_status = kwargs.get("status") - try: - from tools.delegate_tool import ( - SUBAGENT_FAILURE_STATUSES, - format_subagent_failure_line, - ) - if _sub_status in SUBAGENT_FAILURE_STATUSES and ctx._run_still_current(): - _line = format_subagent_failure_line( - kwargs.get("goal"), - _sub_status, - error=kwargs.get("summary") or preview, - duration_seconds=kwargs.get("duration_seconds"), - ) - safe_schedule_threadsafe( - self._runner._deliver_platform_notice(ctx.source, _line), - ctx._loop_for_step, - logger=logger, - log_message="subagent failure notice scheduling error", - ) - except Exception: - logger.debug("subagent failure notice failed", exc_info=True) + self._progress_subagent_notice(preview, kwargs) return - # Live status line (Slack assistant status): stash the tool phrase on the adapter; the - # _keep_typing refresh renders it. Plain dict write, safe from the sync worker thread. - if ( - ctx._live_status_adapter is not None - and ctx._live_status_mode != "off" - and tool_name != "_thinking" - ): - try: - if event_type == "tool.started" and tool_name and ctx._run_still_current(): - from agent.display import build_status_phrase - _phrase = build_status_phrase( - tool_name, - args if ctx._live_status_mode == "full" else None, - ) - ctx._live_status_adapter.set_status_text(ctx.source.chat_id, _phrase) - elif event_type == "tool.completed": - # Between tools the model is genuinely "thinking" - # again — revert to the static default. - ctx._live_status_adapter.set_status_text(ctx.source.chat_id, None) - except Exception as _ls_err: - logger.debug("live status update failed: %s", _ls_err) + self._progress_live_status(event_type, tool_name, args) # "log" mode: append tool.started lines to the log queue, silent in chat. Handled before # the progress_queue guard because log mode runs without a chat progress queue. if ctx.log_queue is not None: @@ -108,249 +114,224 @@ class TurnRunner: return if not ctx.progress_queue or not ctx._run_still_current(): return - - # First-touch onboarding: the first time a tool exceeds _LONG_TOOL_THRESHOLD_S while - # streaming every tool (progress_mode == "all"), append a one-time /verbose hint. if event_type == "tool.completed" and not ctx.long_tool_hint_fired[0]: - try: - duration = kwargs.get("duration") or 0 - if duration >= ctx._LONG_TOOL_THRESHOLD_S and ctx.progress_mode == "all": - from agent.onboarding import ( - TOOL_PROGRESS_FLAG, - is_seen, - mark_seen, - tool_progress_hint_gateway, - ) - _cfg = _load_gateway_config() - gate_on = is_truthy_value( - cfg_get(_cfg, "display", "tool_progress_command"), - default=False, - ) - if gate_on and not is_seen(_cfg, TOOL_PROGRESS_FLAG): - ctx.long_tool_hint_fired[0] = True - ctx.progress_queue.put(tool_progress_hint_gateway()) - mark_seen(_hermes_home / "config.yaml", TOOL_PROGRESS_FLAG) - except Exception as _hint_err: - logger.debug("tool-progress onboarding hint failed: %s", _hint_err) + self._progress_onboarding_hint(kwargs) return - - # "_thinking" is assistant scratch text between tool calls. It is never ordinary tool - # progress: only relay it when the platform explicitly opted into thinking_progress. + # "_thinking" is assistant scratch text between tool calls, never ordinary tool progress: + # only relayed when the platform explicitly opted into thinking_progress. if event_type == "_thinking" or tool_name == "_thinking": - if not ctx._thinking_enabled: - return - thinking_text = preview if tool_name == "_thinking" else tool_name - msg = f"💬 {thinking_text}" if thinking_text else None - if msg: - ctx.progress_queue.put(msg) + if ctx._thinking_enabled: + thinking_text = preview if tool_name == "_thinking" else tool_name + if thinking_text: + ctx.progress_queue.put(f"💬 {thinking_text}") return - # Native task cards consume the ID-bearing tool_start/tool_complete callbacks instead; # name-correlated text events would duplicate cards and mispair concurrent same-tool calls. - if ctx._native_slack_task_cards and event_type in { - "tool.started", - "tool.completed", - }: + if ctx._native_slack_task_cards and event_type in {"tool.started", "tool.completed"}: return - - # If tool_progress is off, only _thinking passes through (above). - # Regular tool calls are suppressed. - if not ctx.tool_progress_enabled: + # tool_progress off → only _thinking passes (above). Only tool.started renders. clarify: + # send_clarify IS the user-facing rendering (a bubble would duplicate it, and verbose mode + # would dump the raw args JSON right under the prompt). Post-`stop`: N parallel tool calls + # fire N tool.started events before the interrupt check, so a late stop must not render them. + if ( + not ctx.tool_progress_enabled + or event_type != "tool.started" + or tool_name == "clarify" + or self._agent_interrupted() + ): return - - # Only act on tool.started events (ignore tool.completed, reasoning.available, etc.) - if event_type not in {"tool.started",}: - return - - # Never render a progress bubble for clarify: send_clarify IS the user-facing rendering, so - # a bubble is duplication, and verbose mode would dump the raw tool-call args JSON, which - # (progress queue drains on a background task) lands right under the rendered prompt. - if tool_name == "clarify": - return - - # Suppress tool-progress bubbles once the user sent `stop`: N parallel tool calls fire N - # "tool.started" events before the interrupt check, so a late `stop` would still render - # all N bubbles. (agent_holder[0] is the shared agent handle across nested scopes.) - try: - _agent_for_interrupt = ctx.agent_holder[0] if ctx.agent_holder else None - if _agent_for_interrupt is not None and getattr( - _agent_for_interrupt, "is_interrupted", False - ): - return - except Exception: - pass - # "new" mode: only report when tool changes if ctx.progress_mode == "new" and tool_name == ctx.last_tool[0]: return ctx.last_tool[0] = tool_name + msg = self._progress_build_message(tool_name, preview, args) + if msg is not None: + self._progress_emit(msg) - # Build progress message with primary argument preview - from agent.display import get_tool_emoji - emoji = get_tool_emoji(tool_name, default="⚙️") - - # Markdown platforms (``supports_code_blocks``) fence terminal commands; plain-text ones - # keep the compact `terminal: "cmd…"` line. No language tag: Slack mrkdwn renders it as a - # literal first code line. Verbose shows the FULL command; "all"/"new" fence but truncate to - # one line capped at ``tool_preview_length`` (default 40), the non-terminal preview budget. - _code_block_full = None - _code_block_short = None + def _progress_subagent_notice(self, preview, kwargs: dict) -> None: + """Only terminal failure statuses render (same notice rail as credit warnings).""" + ctx = self._ctx + status = kwargs.get("status") try: - _progress_adapter = self._runner._adapter_for_source(ctx.source) + from tools.delegate_tool import SUBAGENT_FAILURE_STATUSES, format_subagent_failure_line + if status in SUBAGENT_FAILURE_STATUSES and ctx._run_still_current(): + line = format_subagent_failure_line( + kwargs.get("goal"), status, error=kwargs.get("summary") or preview, + duration_seconds=kwargs.get("duration_seconds"), + ) + self._schedule( + self._runner._deliver_platform_notice(ctx.source, line), + "subagent failure notice scheduling error", + ) except Exception: - _progress_adapter = None - if ( - getattr(_progress_adapter, "supports_code_blocks", False) + logger.debug("subagent failure notice failed", exc_info=True) + + def _progress_live_status(self, event_type: str, tool_name, args) -> None: + """Live status line (Slack assistant status): stash the tool phrase on the adapter; the + _keep_typing refresh renders it. Plain dict write, safe from the sync worker thread.""" + ctx = self._ctx + if ctx._live_status_adapter is None or ctx._live_status_mode == "off" or tool_name == "_thinking": + return + try: + if event_type == "tool.started" and tool_name and ctx._run_still_current(): + from agent.display import build_status_phrase + phrase = build_status_phrase(tool_name, args if ctx._live_status_mode == "full" else None) + ctx._live_status_adapter.set_status_text(ctx.source.chat_id, phrase) + elif event_type == "tool.completed": + # Between tools the model is genuinely "thinking" again — revert to the static default. + ctx._live_status_adapter.set_status_text(ctx.source.chat_id, None) + except Exception as err: + logger.debug("live status update failed: %s", err) + + def _progress_onboarding_hint(self, kwargs: dict) -> None: + """First-touch onboarding: the first time a tool exceeds _LONG_TOOL_THRESHOLD_S while + streaming every tool (progress_mode == "all"), append a one-time /verbose hint.""" + from gateway.run import _hermes_home, _load_gateway_config + ctx = self._ctx + try: + duration = kwargs.get("duration") or 0 + if duration >= ctx._LONG_TOOL_THRESHOLD_S and ctx.progress_mode == "all": + from agent.onboarding import TOOL_PROGRESS_FLAG, is_seen, mark_seen, tool_progress_hint_gateway + cfg = _load_gateway_config() + gate_on = is_truthy_value(cfg_get(cfg, "display", "tool_progress_command"), default=False) + if gate_on and not is_seen(cfg, TOOL_PROGRESS_FLAG): + ctx.long_tool_hint_fired[0] = True + ctx.progress_queue.put(tool_progress_hint_gateway()) + mark_seen(_hermes_home / "config.yaml", TOOL_PROGRESS_FLAG) + except Exception as err: + logger.debug("tool-progress onboarding hint failed: %s", err) + + @staticmethod + def _preview_cap() -> int: + """tool_preview_length (default 40): the one-line preview budget for "all"/"new" modes.""" + from agent.display import get_tool_preview_max_len + pl = get_tool_preview_max_len() + return pl if pl > 0 else 40 + + def _progress_terminal_blocks(self, adapter, tool_name, args, emoji): + """(full, short) fenced blocks for a terminal command on markdown platforms, else (None, None). + + No language tag: Slack mrkdwn renders it as a literal first code line. Verbose shows the FULL + command; "all"/"new" truncate to one line capped at ``tool_preview_length``. Consecutive + terminal calls drop the repeated header so back-to-back commands render as adjacent blocks. + """ + if not ( + getattr(adapter, "supports_code_blocks", False) and tool_name == "terminal" and isinstance(args, dict) and isinstance(args.get("command"), str) and args["command"].strip() ): - from agent.display import get_tool_preview_max_len - _cmd_full = args["command"].rstrip() - # Consecutive terminal calls drop the repeated "💻 terminal" header so back-to-back - # commands render as adjacent code blocks under one header. - _block_header = ( - "" if ctx.last_was_terminal_block[0] else f"{emoji} {tool_name}\n" - ) - _code_block_full = f"{_block_header}```\n{_cmd_full}\n```" - # Single-line, capped preview for non-verbose modes. - _pl = get_tool_preview_max_len() - _cap = _pl if _pl > 0 else 40 - _lines = _cmd_full.splitlines() - _cmd_short = _lines[0] if _lines else _cmd_full - _multiline = len(_lines) > 1 - if len(_cmd_short) > _cap: - _cmd_short = _cmd_short[:_cap - 3] + "..." - elif _multiline: - _cmd_short = _cmd_short + " ..." - _code_block_short = f"{_block_header}```\n{_cmd_short}\n```" + return None, None + cmd_full = args["command"].rstrip() + header = "" if self._ctx.last_was_terminal_block[0] else f"{emoji} {tool_name}\n" + cap = self._preview_cap() + lines = cmd_full.splitlines() + cmd_short = lines[0] if lines else cmd_full + if len(cmd_short) > cap: + cmd_short = cmd_short[:cap - 3] + "..." + elif len(lines) > 1: + cmd_short = cmd_short + " ..." + return f"{header}```\n{cmd_full}\n```", f"{header}```\n{cmd_short}\n```" - # Verbose mode: show detailed arguments, respects tool_preview_length + def _progress_build_message(self, tool_name, preview, args) -> Optional[str]: + """Render the progress line. Verbose mode queues directly (no dedup) and returns None.""" + ctx = self._ctx + from agent.display import get_tool_emoji + emoji = get_tool_emoji(tool_name, default="⚙️") + try: + adapter = self._runner._adapter_for_source(ctx.source) + except Exception: + adapter = None + code_full, code_short = self._progress_terminal_blocks(adapter, tool_name, args, emoji) if ctx.progress_mode == "verbose": - if _code_block_full is not None: + if code_full is not None: ctx.last_was_terminal_block[0] = True - ctx.progress_queue.put(_code_block_full) - return + ctx.progress_queue.put(code_full) + return None ctx.last_was_terminal_block[0] = False if args: from agent.display import get_tool_preview_max_len - _pl = get_tool_preview_max_len() + pl = get_tool_preview_max_len() args_str = json.dumps(args, ensure_ascii=False, default=str) # tool_preview_length 0 (default) = no truncation in verbose mode; the user asked # for full detail and platform message-length limits handle the rest. - if _pl > 0 and len(args_str) > _pl: - args_str = args_str[:_pl - 3] + "..." + if pl > 0 and len(args_str) > pl: + args_str = args_str[:pl - 3] + "..." msg = f"{emoji} {tool_name}({list(args.keys())})\n{args_str}" elif preview: msg = f"{emoji} {tool_name}: \"{preview}\"" else: msg = f"{emoji} {tool_name}..." ctx.progress_queue.put(msg) - return - - # "all" / "new" modes: short preview capped by tool_preview_length (default 40; gateway - # messages persist, unlike CLI spinners). Markdown terminal commands use the fence above. - if _code_block_short is not None: - msg = _code_block_short + return None + if code_short is not None: ctx.last_was_terminal_block[0] = True - elif preview: - from agent.display import ( - get_tool_preview_max_len, - get_tool_verb, - prepare_tool_preview, - tool_verb_connector, - verb_drops_preview, - ) - _pl = get_tool_preview_max_len() - _cap = _pl if _pl > 0 else 40 - _prepared_preview = prepare_tool_preview( - tool_name, - args, - fallback=preview, - max_len=_cap, - ) - if _progress_adapter is not None: - preview = _progress_adapter.format_tool_preview(_prepared_preview) - else: - preview = _prepared_preview.text - # Friendly labels: human-phrased line for built-in tools ("🔍 Searching the web for ...") - # by prefixing the verb onto the computed preview, so the command/url/query is kept. - _verb = get_tool_verb(tool_name) - if _verb: - if verb_drops_preview(tool_name): - msg = f"{emoji} {_verb}" - else: - msg = f"{emoji} {_verb}{tool_verb_connector(tool_name)}{preview}" - else: - msg = f"{emoji} {tool_name}: \"{preview}\"" - ctx.last_was_terminal_block[0] = False - else: - msg = f"{emoji} {tool_name}..." - ctx.last_was_terminal_block[0] = False + return code_short + ctx.last_was_terminal_block[0] = False + if not preview: + return f"{emoji} {tool_name}..." + from agent.display import get_tool_verb, prepare_tool_preview, tool_verb_connector, verb_drops_preview + prepared = prepare_tool_preview(tool_name, args, fallback=preview, max_len=self._preview_cap()) + preview = adapter.format_tool_preview(prepared) if adapter is not None else prepared.text + # Friendly labels: human-phrased line for built-in tools ("🔍 Searching the web for ...") + # by prefixing the verb onto the computed preview, so the command/url/query is kept. + verb = get_tool_verb(tool_name) + if not verb: + return f"{emoji} {tool_name}: \"{preview}\"" + if verb_drops_preview(tool_name): + return f"{emoji} {verb}" + return f"{emoji} {verb}{tool_verb_connector(tool_name)}{preview}" - # Dedup consecutive identical progress messages (common with execute_code: same - # boilerplate imports → identical previews). + def _progress_emit(self, msg: str) -> None: + """Dedup consecutive identical lines (execute_code boilerplate), then route to the native + stream bubble when the consumer accepts tool progress, else the progress queue.""" + ctx = self._ctx + sc = self._stream_consumer() + native = sc is not None and getattr(sc, "accepts_tool_progress", False) if msg == ctx.last_progress_msg[0]: ctx.repeat_count[0] += 1 - # Native-stream-progress routing: dedup updates the last line - # in the overlay rather than sending a queue signal. - _sc = ctx.stream_consumer_holder[0] if ctx.stream_consumer_holder else None - if _sc is not None and getattr(_sc, "accepts_tool_progress", False): - # Replace the last progress line with the dedup version - _sc.on_tool_progress(f"{msg} (×{ctx.repeat_count[0] + 1})") - return - # Update the last line in progress_lines with a counter - # via a special "dedup" queue message. - ctx.progress_queue.put(("__dedup__", msg, ctx.repeat_count[0])) + if native: + sc.on_tool_progress(f"{msg} (×{ctx.repeat_count[0] + 1})") + else: + ctx.progress_queue.put(("__dedup__", msg, ctx.repeat_count[0])) return ctx.last_progress_msg[0] = msg ctx.repeat_count[0] = 0 + if native: + sc.on_tool_progress(msg) + else: + ctx.progress_queue.put(msg) - # If the stream consumer is active with native streaming, inject progress into the stream - # bubble instead of the separate progress queue. - _sc = ctx.stream_consumer_holder[0] if ctx.stream_consumer_holder else None - if _sc is not None and getattr(_sc, "accepts_tool_progress", False): - _sc.on_tool_progress(msg) - return + # ── Slack-native task cards (progress-queue drain) ────────────────────────────────────── - ctx.progress_queue.put(msg) - - async def _send_native_task_card_progress(self, adapter) -> None: - """Drain the progress queue into Slack-native plan/task cards. - - On any native failure, fall back to an editable in-thread message so progress stays live. - """ - ctx = self._ctx - tasks: Dict[str, Dict[str, str]] = {} - task_order: List[str] = [] + @dataclasses.dataclass + class _TaskCardState: + """Task-card rail state for ``_send_native_task_card_progress``.""" + adapter: Any + tasks: Dict[str, Dict[str, str]] = dataclasses.field(default_factory=dict) + task_order: List[str] = dataclasses.field(default_factory=list) fallback_msg_id: Optional[str] = None - native_failed = False - anonymous_seq = 0 + native_failed: bool = False + anonymous_seq: int = 0 + @staticmethod def _compact(value: Any, limit: int = 120) -> str: text = re.sub(r"\s+", " ", str(value or "")).strip() - if len(text) <= limit: - return text - return text[: limit - 3].rstrip() + "..." + return text if len(text) <= limit else text[: limit - 3].rstrip() + "..." - def _visible_tasks() -> List[Dict[str, str]]: - return [tasks[task_id] for task_id in task_order[-8:]] + def visible_tasks(self) -> List[Dict[str, str]]: + return [self.tasks[task_id] for task_id in self.task_order[-8:]] - def _fallback_text() -> str: - labels = { - "in_progress": "running", - "complete": "complete", - "error": "error", - } + def fallback_text(self) -> str: + labels = {"in_progress": "running", "complete": "complete", "error": "error"} lines = [ f"- {task['title']} - {labels.get(task['status'], task['status'])}" - for task in _visible_tasks() + for task in self.visible_tasks() ] return "Hermes is working\n" + "\n".join(lines) - def _apply_native_event(raw: Any) -> bool: - nonlocal anonymous_seq + def apply_event(self, raw: Any) -> bool: if not isinstance(raw, dict): return False event_type = raw.get("type") @@ -358,113 +339,78 @@ class TurnRunner: return False call_id = str(raw.get("tool_call_id") or "") if not call_id: - anonymous_seq += 1 - call_id = f"anonymous_{anonymous_seq}" + self.anonymous_seq += 1 + call_id = f"anonymous_{self.anonymous_seq}" tool_name = str(raw.get("tool_name") or "tool") - if event_type == "tool.started": - title = tool_name - preview = _compact(raw.get("preview"), 64) - if preview: - title = f"{tool_name} - {preview}" - if call_id not in tasks: - task_order.append(call_id) - tasks[call_id] = { - "id": call_id, - "title": _compact(title), - "status": "in_progress", - } + preview = self._compact(raw.get("preview"), 64) + title = f"{tool_name} - {preview}" if preview else tool_name + if call_id not in self.tasks: + self.task_order.append(call_id) + self.tasks[call_id] = {"id": call_id, "title": self._compact(title), "status": "in_progress"} return True - - task = tasks.get(call_id) + task = self.tasks.get(call_id) if task is None: # Completion-only events are rare but valid on some runtimes; keep their real ID # instead of guessing a same-name pending call. - task = { - "id": call_id, - "title": _compact(tool_name), - "status": "in_progress", - } - tasks[call_id] = task - task_order.append(call_id) + task = {"id": call_id, "title": self._compact(tool_name), "status": "in_progress"} + self.tasks[call_id] = task + self.task_order.append(call_id) task["status"] = "error" if raw.get("is_error") else "complete" return True - async def _send_or_edit_fallback() -> None: - nonlocal fallback_msg_id - text = _fallback_text() - if fallback_msg_id: - result = await adapter.edit_message( - chat_id=ctx.source.chat_id, - message_id=fallback_msg_id, - content=text, - metadata=ctx._progress_metadata, - ) - if getattr(result, "success", False): - return - result = await adapter.send( - chat_id=ctx.source.chat_id, - content=text, - reply_to=ctx._progress_reply_to, + async def _task_card_send_or_edit_fallback(self, st) -> None: + ctx = self._ctx + text = st.fallback_text() + if st.fallback_msg_id: + result = await st.adapter.edit_message( + chat_id=ctx.source.chat_id, message_id=st.fallback_msg_id, content=text, metadata=ctx._progress_metadata, ) - if getattr(result, "success", False) and getattr( - result, "message_id", None - ): - fallback_msg_id = str(result.message_id) - if ctx._cleanup_progress: - ctx._cleanup_msg_ids.append(fallback_msg_id) - - async def _publish_native_progress() -> None: - nonlocal native_failed - if not tasks: + if getattr(result, "success", False): return - if not native_failed: - result = await adapter.send_native_task_card_progress( - chat_id=ctx.source.chat_id, - tasks=_visible_tasks(), - title="Hermes is working", - reply_to=ctx._progress_reply_to, - metadata=ctx._progress_metadata, - fallback_text=_fallback_text(), - ) - if getattr(result, "success", False): - return - native_failed = True - logger.warning( - "Slack native task-card progress failed; falling back " - "to an editable text update: %s", - getattr(result, "error", "unknown error"), - ) - # Once the native rail fails, every later lifecycle event - # edits the same fallback message so progress remains live. - await _send_or_edit_fallback() + result = await self._send_progress_text(st, text) + if getattr(result, "success", False) and getattr(result, "message_id", None): + st.fallback_msg_id = str(result.message_id) - def _drain_native_queue() -> bool: - changed = False - while True: - try: - changed = _apply_native_event( - ctx.progress_queue.get_nowait() - ) or changed - except queue.Empty: - return changed - except Exception: - logger.debug( - "Slack native progress queue drain failed", - exc_info=True, - ) - return changed + async def _task_card_publish(self, st) -> None: + ctx = self._ctx + if not st.tasks: + return + if not st.native_failed: + result = await st.adapter.send_native_task_card_progress( + chat_id=ctx.source.chat_id, tasks=st.visible_tasks(), title="Hermes is working", + reply_to=ctx._progress_reply_to, metadata=ctx._progress_metadata, + fallback_text=st.fallback_text(), + ) + if getattr(result, "success", False): + return + st.native_failed = True + logger.warning( + "Slack native task-card progress failed; falling back " + "to an editable text update: %s", getattr(result, "error", "unknown error"), + ) + # Once the native rail fails, every later lifecycle event edits the same fallback message. + await self._task_card_send_or_edit_fallback(st) - def _agent_interrupted() -> bool: + def _task_card_drain(self, st) -> bool: + changed = False + while True: try: - _agent = ctx.agent_holder[0] if ctx.agent_holder else None - return bool( - _agent is not None and getattr(_agent, "is_interrupted", False) - ) + changed = st.apply_event(self._ctx.progress_queue.get_nowait()) or changed + except queue.Empty: + return changed except Exception: - return False + logger.debug("Slack native progress queue drain failed", exc_info=True) + return changed + async def _send_native_task_card_progress(self, adapter) -> None: + """Drain the progress queue into Slack-native plan/task cards. + + On any native failure, fall back to an editable in-thread message so progress stays live. + """ + ctx = self._ctx + st = self._TaskCardState(adapter) try: while True: if not ctx._run_still_current(): @@ -474,16 +420,13 @@ class TurnRunner: except queue.Empty: await asyncio.sleep(0.1) continue - - if _agent_interrupted(): + if self._agent_interrupted(): continue - - if _apply_native_event(raw): - await _publish_native_progress() + if st.apply_event(raw): + await self._task_card_publish(st) except asyncio.CancelledError: - if _drain_native_queue() and ctx._run_still_current(): - if not _agent_interrupted(): - await _publish_native_progress() + if self._task_card_drain(st) and ctx._run_still_current() and not self._agent_interrupted(): + await self._task_card_publish(st) return finally: if hasattr(adapter, "stop_native_task_card_progress"): @@ -491,17 +434,15 @@ class TurnRunner: # final-delivery logic (cleanup awaits catch only CancelledError). try: await adapter.stop_native_task_card_progress( - ctx.source.chat_id, - reply_to=ctx._progress_reply_to, + ctx.source.chat_id, reply_to=ctx._progress_reply_to, metadata=ctx._progress_metadata, ) except asyncio.CancelledError: raise except Exception: - logger.debug( - "task-card stop failed during turn cleanup", - exc_info=True, - ) + logger.debug("task-card stop failed during turn cleanup", exc_info=True) + + # ── editable progress bubbles (progress-queue drain) ──────────────────────────────────── @dataclasses.dataclass class _ProgressEditState: @@ -516,75 +457,53 @@ class TurnRunner: def _progress_edit_state(self, adapter) -> "TurnRunner._ProgressEditState": ctx = self._ctx - progress_lines = [] # Accumulated tool lines for the CURRENT editable bubble - progress_msg_id = None # ID of the current progress message to edit - can_edit = ctx.progress_grouping != "separate" # "separate" = one message per tool (pre-v0.9 behavior) - - _progress_len_fn = ( - adapter.message_len_fn - if isinstance(adapter, BasePlatformAdapter) - else len - ) + len_fn = adapter.message_len_fn if isinstance(adapter, BasePlatformAdapter) else len try: - _raw_progress_limit = int(getattr(adapter, "MAX_MESSAGE_LENGTH", 4000) or 4000) + raw_limit = int(getattr(adapter, "MAX_MESSAGE_LENGTH", 4000) or 4000) except Exception: - _raw_progress_limit = 4000 + raw_limit = 4000 # Per-chat resolution (relay adapter fronting N platforms): cap and length unit follow the # chat's underlying platform; native adapters return their scalar/property unchanged. if isinstance(adapter, BasePlatformAdapter): try: - _raw_progress_limit = int( - adapter.max_message_length_for_chat(ctx.source.chat_id) or 4000 - ) - _progress_len_fn = adapter.message_len_fn_for_chat(ctx.source.chat_id) + raw_limit = int(adapter.max_message_length_for_chat(ctx.source.chat_id) or 4000) + len_fn = adapter.message_len_fn_for_chat(ctx.source.chat_id) except Exception: pass - # Leave a little room for platform quirks / formatting. For tiny - # test adapters keep the limit usable instead of clamping to 500+. - _PROGRESS_TEXT_LIMIT = max( - 1, - _raw_progress_limit - (64 if _raw_progress_limit > 128 else 0), - ) - - # Detect whether the adapter's edit_message accepts metadata so - # overflow edits preserve Telegram topic/thread routing (#27487). - _edit_accepts_metadata = False + # Detect whether edit_message accepts metadata so overflow edits preserve Telegram + # topic/thread routing. + edit_accepts_metadata = False if ctx._progress_metadata: try: - _edit_params = inspect.signature(adapter.edit_message).parameters - _edit_accepts_metadata = ( - "metadata" in _edit_params - or any( - param.kind is inspect.Parameter.VAR_KEYWORD - for param in _edit_params.values() - ) + params = inspect.signature(adapter.edit_message).parameters + edit_accepts_metadata = "metadata" in params or any( + p.kind is inspect.Parameter.VAR_KEYWORD for p in params.values() ) except (TypeError, ValueError): - _edit_accepts_metadata = False + edit_accepts_metadata = False return self._ProgressEditState( adapter=adapter, - progress_lines=progress_lines, - progress_msg_id=progress_msg_id, - can_edit=can_edit, - _progress_len_fn=_progress_len_fn, - _PROGRESS_TEXT_LIMIT=_PROGRESS_TEXT_LIMIT, - _edit_accepts_metadata=_edit_accepts_metadata, + progress_lines=[], + progress_msg_id=None, + # "separate" = one message per tool (pre-v0.9 behavior) + can_edit=ctx.progress_grouping != "separate", + _progress_len_fn=len_fn, + # Leave room for platform quirks / formatting; tiny test adapters keep a usable limit. + _PROGRESS_TEXT_LIMIT=max(1, raw_limit - (64 if raw_limit > 128 else 0)), + _edit_accepts_metadata=edit_accepts_metadata, ) async def _edit_progress_message(self, st, message_id: str, content: str): ctx = self._ctx - kwargs = { - "chat_id": ctx.source.chat_id, - "message_id": message_id, - "content": content, - } + kwargs = {"chat_id": ctx.source.chat_id, "message_id": message_id, "content": content} if getattr(st.adapter, "REQUIRES_EDIT_FINALIZE", False): kwargs["finalize"] = True if st._edit_accepts_metadata: kwargs["metadata"] = ctx._progress_metadata return await st.adapter.edit_message(**kwargs) - def _progress_text(self, lines: list) -> str: + @staticmethod + def _progress_text(lines: list) -> str: return "\n".join(str(line) for line in lines) def _split_progress_groups(self, st, lines: list) -> list[list]: @@ -602,21 +521,10 @@ class TurnRunner: groups.append(current) return groups - def _track_progress_result(self, result) -> None: - ctx = self._ctx - if ( - ctx._cleanup_progress - and getattr(result, "success", False) - and getattr(result, "message_id", None) - ): - ctx._cleanup_msg_ids.append(str(result.message_id)) - async def _send_progress_text(self, st, text: str): ctx = self._ctx result = await st.adapter.send( - chat_id=ctx.source.chat_id, - content=text, - reply_to=ctx._progress_reply_to, + chat_id=ctx.source.chat_id, content=text, reply_to=ctx._progress_reply_to, metadata=ctx._progress_metadata, ) self._track_progress_result(result) @@ -625,239 +533,170 @@ class TurnRunner: async def _roll_progress_overflow_if_needed(self, st) -> bool: """Start fresh editable progress bubbles before a bubble exceeds limit. - Returns True when it delivered/split the buffer or a transient edit failure left it - intact for retry — either way the caller skips the normal send/edit path this tick. - """ + Returns True when it delivered/split the buffer or a transient edit failure left it + intact for retry — either way the caller skips the normal send/edit path this tick. + """ if not st.progress_lines or not st.can_edit: return False groups = self._split_progress_groups(st, st.progress_lines) if len(groups) <= 1: return False - first_text = self._progress_text(groups[0]) if st.progress_msg_id is not None: result = await self._edit_progress_message(st, st.progress_msg_id, first_text) if not result.success: if getattr(result, "retryable", False): - logger.debug( - "[%s] Transient overflow edit failure — keeping can_edit=True", - st.adapter.name, - ) + logger.debug("[%s] Transient overflow edit failure — keeping can_edit=True", st.adapter.name) return True st.can_edit = False - # Fall back to the existing non-edit behavior below. + # Fall back to the existing non-edit behavior. return False else: result = await self._send_progress_text(st, first_text) if result.success and result.message_id: st.progress_msg_id = result.message_id - for group in groups[1:]: result = await self._send_progress_text(st, self._progress_text(group)) if result.success and result.message_id: st.progress_msg_id = result.message_id - # The newest continuation is the only mutable bubble: keep just its lines so later # edits update it instead of replaying the full transcript into new messages. st.progress_lines = groups[-1] return True + @staticmethod + def _is_reset_marker(raw) -> bool: + return isinstance(raw, tuple) and len(raw) >= 1 and raw[0] == "__reset__" + + def _reset_progress_bubble(self, st) -> None: + """Content bubble landed — close the tool-progress bubble so the next tool starts fresh + below it; else tool edits hit the ORIGINAL message above (out of order).""" + st.progress_msg_id = None + st.progress_lines = [] + self._ctx.last_progress_msg[0] = None + self._ctx.repeat_count[0] = 0 + + def _progress_absorb(self, st, raw) -> Any: + """Fold a queue item into the bubble buffer; returns the line to render this tick.""" + if isinstance(raw, tuple) and len(raw) == 3 and raw[0] == "__dedup__": + _, base_msg, count = raw + if st.progress_lines: + st.progress_lines[-1] = f"{base_msg} (×{count + 1})" + return st.progress_lines[-1] + return base_msg + st.progress_lines.append(raw) + return raw + + async def _flush_progress_edit(self, st) -> None: + if st.can_edit and st.progress_lines and st.progress_msg_id: + with suppress(Exception): + await self._edit_progress_message(st, st.progress_msg_id, self._progress_text(st.progress_lines)) + async def _drain_progress_on_cancel(self, st) -> None: ctx = self._ctx - # Drain remaining queued messages while not ctx.progress_queue.empty(): try: raw = ctx.progress_queue.get_nowait() - if isinstance(raw, tuple) and len(raw) == 3 and raw[0] == "__dedup__": - _, base_msg, count = raw - if st.progress_lines: - st.progress_lines[-1] = f"{base_msg} (×{count + 1})" - await self._roll_progress_overflow_if_needed(st) - elif isinstance(raw, tuple) and len(raw) >= 1 and raw[0] == "__reset__": + if self._is_reset_marker(raw): # Content-bubble marker during drain: close the current progress bubble # and start a fresh one for tool lines that arrived after. await self._roll_progress_overflow_if_needed(st) - if st.can_edit and st.progress_lines and st.progress_msg_id: - _pending_text = self._progress_text(st.progress_lines) - with suppress(Exception): - await self._edit_progress_message(st, st.progress_msg_id, _pending_text) - st.progress_msg_id = None - st.progress_lines = [] - ctx.last_progress_msg[0] = None - ctx.repeat_count[0] = 0 + await self._flush_progress_edit(st) + self._reset_progress_bubble(st) else: - st.progress_lines.append(raw) + self._progress_absorb(st, raw) await self._roll_progress_overflow_if_needed(st) except Exception: break # Final edit with all remaining tools (only if editing works) if st.can_edit and st.progress_lines and st.progress_msg_id: await self._roll_progress_overflow_if_needed(st) - if st.can_edit and st.progress_lines and st.progress_msg_id: - full_text = self._progress_text(st.progress_lines) - with suppress(Exception): - await self._edit_progress_message(st, st.progress_msg_id, full_text) + await self._flush_progress_edit(st) + + async def _progress_restore_typing(self, st) -> None: + ctx = self._ctx + await asyncio.sleep(0.3) + if ctx._run_still_current(): + await st.adapter.send_typing(ctx.source.chat_id, metadata=ctx._progress_metadata) + + async def _progress_send_or_edit(self, st, msg) -> bool: + """Deliver this tick's bubble. Returns False on a transient edit failure (retry next tick). + + Transient network errors (ConnectError, timeouts) must not disable editing; only permanent + failures (not found, permissions) set can_edit=False. Flood control backs off but keeps editing. + """ + if st.can_edit and st.progress_msg_id is not None: + result = await self._edit_progress_message(st, st.progress_msg_id, "\n".join(st.progress_lines)) + if result.success: + return True + if getattr(result, "retryable", False): + logger.debug("[%s] Transient edit failure — keeping can_edit=True", st.adapter.name) + return False + err = (getattr(result, "error", "") or "").lower() + if "flood" in err or "retry after" in err: + logger.info("[%s] Progress edit flood control, backing off", st.adapter.name) + else: + st.can_edit = False + await self._send_progress_text(st, msg) + return True + # First tool: send all accumulated text as a new message; editing unsupported: just this line. + result = await self._send_progress_text(st, "\n".join(st.progress_lines) if st.can_edit else msg) + if result.success and result.message_id: + st.progress_msg_id = result.message_id + return True async def send_progress_messages(self): ctx = self._ctx if not ctx.progress_queue: return - adapter = self._runner._adapter_for_source(ctx.source) if not adapter: return - - if ctx._native_slack_task_cards and hasattr( - adapter, "send_native_task_card_progress" - ): + if ctx._native_slack_task_cards and hasattr(adapter, "send_native_task_card_progress"): await self._send_native_task_card_progress(adapter) return - # Skip tool progress for platforms that can't edit messages (e.g. iMessage/BlueBubbles): # each update would be a separate bubble. getattr, not attribute access: duck-typed # adapters (test fakes, minimal plugins) may lack edit_message — treated as "can't edit". - _adapter_edit = getattr(type(adapter), "edit_message", None) - if _adapter_edit is None or _adapter_edit is BasePlatformAdapter.edit_message: - while not ctx.progress_queue.empty(): - try: - ctx.progress_queue.get_nowait() - except Exception: - break + adapter_edit = getattr(type(adapter), "edit_message", None) + if adapter_edit is None or adapter_edit is BasePlatformAdapter.edit_message: + self._drain_progress_queue() return - st = self._progress_edit_state(adapter) - _last_edit_ts = 0.0 # Throttle edits to avoid Telegram flood control - _PROGRESS_EDIT_INTERVAL = 1.5 # Minimum seconds between edits - + last_edit_ts = 0.0 + EDIT_INTERVAL = 1.5 # Minimum seconds between edits (Telegram flood control) while True: try: if not ctx._run_still_current(): - while not ctx.progress_queue.empty(): - try: - ctx.progress_queue.get_nowait() - except Exception: - break + self._drain_progress_queue() return - raw = ctx.progress_queue.get_nowait() - # Drain silently when interrupted: events queued in the window between tool parse # and interrupt processing should not render as bubbles. - try: - _agent_for_interrupt = ctx.agent_holder[0] if ctx.agent_holder else None - if _agent_for_interrupt is not None and getattr( - _agent_for_interrupt, "is_interrupted", False - ): - # Drop this event and continue draining. - await asyncio.sleep(0) - continue - except Exception: - pass - - # Handle dedup messages: update last line with repeat counter - if isinstance(raw, tuple) and len(raw) == 3 and raw[0] == "__dedup__": - _, base_msg, count = raw - if st.progress_lines: - st.progress_lines[-1] = f"{base_msg} (×{count + 1})" - msg = st.progress_lines[-1] if st.progress_lines else base_msg - elif isinstance(raw, tuple) and len(raw) >= 1 and raw[0] == "__reset__": - # Content bubble landed — close the tool-progress bubble so the next tool starts - # fresh below it; else tool edits hit the ORIGINAL message above (out of order). - st.progress_msg_id = None - st.progress_lines = [] - ctx.last_progress_msg[0] = None - ctx.repeat_count[0] = 0 + if self._agent_interrupted(): + await asyncio.sleep(0) continue - else: - msg = raw - st.progress_lines.append(msg) - + if self._is_reset_marker(raw): + self._reset_progress_bubble(st) + continue + msg = self._progress_absorb(st, raw) if await self._roll_progress_overflow_if_needed(st): - _last_edit_ts = time.monotonic() - await asyncio.sleep(0.3) - if ctx._run_still_current(): - await st.adapter.send_typing(ctx.source.chat_id, metadata=ctx._progress_metadata) + last_edit_ts = time.monotonic() + await self._progress_restore_typing(st) continue - - # Throttle edits: batch rapid tool updates into fewer API calls to avoid Telegram - # flood control (grammY pattern: proactively rate-limit rather than react to 429s). - _now = time.monotonic() - _remaining = _PROGRESS_EDIT_INTERVAL - (_now - _last_edit_ts) - if _remaining > 0: - # Wait out the throttle interval, then loop back to drain any further queued - # messages before sending a single batched edit. - await asyncio.sleep(_remaining) + # Throttle edits: batch rapid tool updates into fewer API calls (grammY pattern: + # proactively rate-limit rather than react to 429s). Loop back to drain further + # queued messages before sending a single batched edit. + remaining = EDIT_INTERVAL - (time.monotonic() - last_edit_ts) + if remaining > 0: + await asyncio.sleep(remaining) continue - if not ctx._run_still_current(): return - - if st.can_edit and st.progress_msg_id is not None: - # Try to edit the existing progress message - full_text = "\n".join(st.progress_lines) - result = await self._edit_progress_message(st, st.progress_msg_id, full_text) - if not result.success: - _err = (getattr(result, "error", "") or "").lower() - # Transient network errors (ConnectError, timeouts) must not disable editing; - # only permanent failures (flood, not found, permissions) set can_edit = False. - if getattr(result, "retryable", False): - logger.debug( - "[%s] Transient edit failure — keeping can_edit=True", - st.adapter.name, - ) - continue - if "flood" in _err or "retry after" in _err: - # Flood control hit — backoff but keep editing. - # Only disable edits for non-recoverable errors. - logger.info( - "[%s] Progress edit flood control, backing off", - st.adapter.name, - ) - _last_edit_ts = time.monotonic() - else: - st.can_edit = False - _flood_result = await st.adapter.send( - chat_id=ctx.source.chat_id, - content=msg, - reply_to=ctx._progress_reply_to, - metadata=ctx._progress_metadata, - ) - if ( - ctx._cleanup_progress - and getattr(_flood_result, "success", False) - and getattr(_flood_result, "message_id", None) - ): - ctx._cleanup_msg_ids.append(str(_flood_result.message_id)) - else: - if st.can_edit: - # First tool: send all accumulated text as new message - full_text = "\n".join(st.progress_lines) - result = await st.adapter.send( - chat_id=ctx.source.chat_id, - content=full_text, - reply_to=ctx._progress_reply_to, - metadata=ctx._progress_metadata, - ) - else: - # Editing unsupported: send just this line - result = await st.adapter.send( - chat_id=ctx.source.chat_id, - content=msg, - reply_to=ctx._progress_reply_to, - metadata=ctx._progress_metadata, - ) - if result.success and result.message_id: - st.progress_msg_id = result.message_id - if ctx._cleanup_progress: - ctx._cleanup_msg_ids.append(str(result.message_id)) - - _last_edit_ts = time.monotonic() - - # Restore typing indicator - await asyncio.sleep(0.3) - if ctx._run_still_current(): - await st.adapter.send_typing(ctx.source.chat_id, metadata=ctx._progress_metadata) - + if not await self._progress_send_or_edit(st, msg): + continue + last_edit_ts = time.monotonic() + await self._progress_restore_typing(st) except queue.Empty: await asyncio.sleep(0.3) except asyncio.CancelledError: @@ -867,79 +706,59 @@ class TurnRunner: logger.error("Progress message error: %s", e) await asyncio.sleep(1) + # ── ID-bearing lifecycle callbacks (agent thread) ─────────────────────────────────────── + def voice_ack_callback(self, call_id, tool_name, args): """tool_start_callback: speak a one-time ack in the voice channel.""" - from gateway.run import safe_schedule_threadsafe ctx = self._ctx - if ctx._voice_ack_fired[0] or ctx._voice_ack_guild[0] is None: - return - if not ctx._run_still_current(): + if ctx._voice_ack_fired[0] or ctx._voice_ack_guild[0] is None or not ctx._run_still_current(): return ctx._voice_ack_fired[0] = True - _adapter = self._runner.adapters.get(Platform.DISCORD) - if _adapter is None or not hasattr(_adapter, "play_ack_in_voice"): + adapter = self._runner.adapters.get(Platform.DISCORD) + if adapter is None or not hasattr(adapter, "play_ack_in_voice"): return try: - safe_schedule_threadsafe( - _adapter.play_ack_in_voice(ctx._voice_ack_guild[0]), - ctx._voice_ack_loop, - logger=logger, - log_message="voice ack scheduling error", + self._schedule( + adapter.play_ack_in_voice(ctx._voice_ack_guild[0]), "voice ack scheduling error", + loop=ctx._voice_ack_loop, ) - except Exception as _ack_err: - logger.debug("voice ack schedule failed: %s", _ack_err) + except Exception as err: + logger.debug("voice ack schedule failed: %s", err) - # ── Slack-native task cards: ID-bearing lifecycle callbacks ── ride agent.tool_start_callback / - # agent.tool_complete_callback so start/completion correlate by the REAL tool-call id; the - # name-correlated progress_callback text events would duplicate cards and mispair concurrent calls. + # Slack-native task cards ride agent.tool_start_callback / tool_complete_callback so start and + # completion correlate by the REAL tool-call id; name-correlated progress_callback text events + # would duplicate cards and mispair concurrent calls. + + def _native_card_gate(self) -> bool: + ctx = self._ctx + return bool(ctx.progress_queue) and ctx._run_still_current() and not self._agent_interrupted() def native_tool_start_callback(self, call_id, tool_name, args): """Queue an ID-correlated native progress start from the agent thread.""" - ctx = self._ctx - if not ctx.progress_queue or not ctx._run_still_current(): + if not self._native_card_gate(): return - try: - _agent = ctx.agent_holder[0] if ctx.agent_holder else None - if _agent is not None and getattr(_agent, "is_interrupted", False): - return - except Exception: - pass from agent.display import build_tool_preview - - ctx.progress_queue.put( - { - "type": "tool.started", - "tool_call_id": str(call_id or ""), - "tool_name": str(tool_name or "tool"), - "preview": build_tool_preview( - str(tool_name or "tool"), args or {}, max_len=64 - ) - or "", - } - ) + name = str(tool_name or "tool") + self._ctx.progress_queue.put({ + "type": "tool.started", + "tool_call_id": str(call_id or ""), + "tool_name": name, + "preview": build_tool_preview(name, args or {}, max_len=64) or "", + }) def native_tool_complete_callback(self, call_id, tool_name, args, result): """Queue the matching native completion using the real tool-call ID.""" - ctx = self._ctx - if not ctx.progress_queue or not ctx._run_still_current(): + if not self._native_card_gate(): return - try: - _agent = ctx.agent_holder[0] if ctx.agent_holder else None - if _agent is not None and getattr(_agent, "is_interrupted", False): - return - except Exception: - pass from agent.display import _detect_tool_failure - - is_error, _ = _detect_tool_failure(str(tool_name or "tool"), result) - ctx.progress_queue.put( - { - "type": "tool.completed", - "tool_call_id": str(call_id or ""), - "tool_name": str(tool_name or "tool"), - "is_error": bool(is_error), - } - ) + name = str(tool_name or "tool") + is_error, _ = _detect_tool_failure(name, result) + self._ctx.progress_queue.put({ + "type": "tool.completed", + "tool_call_id": str(call_id or ""), + "tool_name": name, + "is_error": bool(is_error), + }) def combined_tool_start_callback(self, call_id, tool_name, args): """Compose the voice ack + native task-card start consumers.""" @@ -949,42 +768,41 @@ class TurnRunner: if ctx._native_slack_task_cards: self.native_tool_start_callback(call_id, tool_name, args) + # ── hook / status bridges (agent thread → gateway loop) ──────────────────────────────── + def _step_callback_sync(self, iteration: int, prev_tools: list) -> None: - from gateway.run import safe_schedule_threadsafe ctx = self._ctx if not ctx._run_still_current(): return # prev_tools may be list[str] or list[dict] with "name"/"result" keys. Normalise so # "tool_names" stays backward-compatible for user hooks that do ', '.join(tool_names). - _names: list[str] = [] - for _t in (prev_tools or []): - if isinstance(_t, dict): - _names.append(_t.get("name") or "") - else: - _names.append(str(_t)) - safe_schedule_threadsafe( + names = [(t.get("name") or "") if isinstance(t, dict) else str(t) for t in (prev_tools or [])] + self._schedule( ctx._hooks_ref.emit("agent:step", { "platform": ctx.source.platform.value if ctx.source.platform else "", "user_id": ctx.source.user_id, "session_id": ctx.session_id, "iteration": iteration, - "tool_names": _names, + "tool_names": names, "tools": prev_tools, }), - ctx._loop_for_step, - logger=logger, - log_message="agent:step hook scheduling error", + "agent:step hook scheduling error", ) def _event_callback_sync(self, event_type: str, context: dict) -> None: ctx = self._ctx try: - asyncio.run_coroutine_threadsafe( - ctx._hooks_ref.emit(event_type, context), - ctx._loop_for_step, - ) - except Exception as _e: - logger.debug("event_callback hook error: %s", _e) + asyncio.run_coroutine_threadsafe(ctx._hooks_ref.emit(event_type, context), ctx._loop_for_step) + except Exception as e: + logger.debug("event_callback hook error: %s", e) + + def _status_live(self) -> bool: + """Status adapter present and this run is still the current generation.""" + return bool(self._ctx._status_adapter) and self._ctx._run_still_current() + + def _send_status_text(self, text: str, metadata, log_message: str) -> None: + ctx = self._ctx + self._schedule(ctx._status_adapter.send(ctx._status_chat_id, text, metadata=metadata), log_message) def _attach_session_title_callback(self, agent, ctx) -> None: """Wire the platform thread-rename lane onto the agent as `_on_session_title`. @@ -995,928 +813,717 @@ class TurnRunner: # Gateway auto-title failures are not user-actionable, so never surface them as messages; # overriding the failure sink keeps CLI on _emit_auxiliary_failure while gateway logs debug. def _title_failure_cb(task: str, exc: BaseException) -> None: - logger.debug( - "Gateway auto-title failure suppressed (not user-visible): %s: %s", - task, exc, - ) + logger.debug("Gateway auto-title failure suppressed (not user-visible): %s: %s", task, exc) agent._title_failure_callback = _title_failure_cb - session_id = getattr(agent, "session_id", None) source = ctx.source - + runner = self._runner # Both lanes spend a rate-limited platform call per title, so they use the model's title # only (TitleCallback); renaming twice burns Discord's 2-per-10-min budget on a throwaway. - if self._runner._is_telegram_topic_lane(source): - agent._on_session_title = lambda title, title_source: ( - title_source == "llm" - and self._runner._schedule_telegram_topic_title_rename( - source, session_id, title, - ) - ) - elif self._runner._is_discord_auto_thread_lane(source) or ( - self._runner._is_relay_discord_channel_lane(source) - ): - # Relay note: the second predicate is shape-only (relay Discord channel event). - # Whether the connector auto-threaded our reply is only knowable AFTER delivery, so - # the callback must be registered eagerly and the rename lane does the cache lookup - # at fire time — gating registration on the cache read meant it never registered. - agent._on_session_title = lambda title, title_source: ( - title_source == "llm" - and self._runner._schedule_discord_semantic_thread_rename( - source, session_id, title, - ) - ) + # Relay Discord predicate is shape-only: whether the connector auto-threaded our reply is + # only knowable AFTER delivery, so register eagerly and let the rename lane look up the + # cache at fire time — gating registration on the cache read meant it never registered. + if runner._is_telegram_topic_lane(source): + lane = "_schedule_telegram_topic_title_rename" + elif runner._is_discord_auto_thread_lane(source) or runner._is_relay_discord_channel_lane(source): + lane = "_schedule_discord_semantic_thread_rename" + else: + return + agent._on_session_title = lambda title, title_source: ( + title_source == "llm" and getattr(runner, lane)(source, session_id, title) + ) except Exception: logger.debug("Failed to attach session title callback", exc_info=True) def _status_callback_sync(self, event_type: str, message: str) -> None: from gateway.run import ( - _prepare_gateway_status_message, - _redact_gateway_user_facing_secrets, + _prepare_gateway_status_message, _redact_gateway_user_facing_secrets, _send_or_update_status_coro, - safe_schedule_threadsafe, ) ctx = self._ctx - if not ctx._status_adapter or not ctx._run_still_current(): + if not self._status_live(): return - prepared_message = _prepare_gateway_status_message( - ctx.source.platform, - event_type, - message, - ) - if prepared_message is None: + prepared = _prepare_gateway_status_message(ctx.source.platform, event_type, message) + if prepared is None: logger.debug( "status_callback suppressed for %s/%s: %s", - ctx.source.platform.value if ctx.source.platform else "unknown", - event_type, + ctx.source.platform.value if ctx.source.platform else "unknown", event_type, _redact_gateway_user_facing_secrets(str(message or ""))[:160], ) return - _fut = safe_schedule_threadsafe( - _send_or_update_status_coro(ctx._status_adapter, ctx._status_chat_id, event_type, prepared_message, ctx._status_thread_metadata), - ctx._loop_for_step, - logger=logger, - log_message=f"status_callback ({event_type}) scheduling error", + fut = self._schedule( + _send_or_update_status_coro( + ctx._status_adapter, ctx._status_chat_id, event_type, prepared, ctx._status_thread_metadata, + ), + f"status_callback ({event_type}) scheduling error", ) - if _fut is None: - return - if ctx._cleanup_progress: - def _track_status_id(fut) -> None: - try: - res = fut.result() - except Exception: - return - mid = getattr(res, "message_id", None) - if getattr(res, "success", False) and mid: - ctx._cleanup_msg_ids.append(str(mid)) - _fut.add_done_callback(_track_status_id) + if fut is not None and ctx._cleanup_progress: + fut.add_done_callback(self._track_future_cleanup_id) + + # ── stream consumer / interim commentary wiring ───────────────────────────────────────── def _setup_stream_consumer(self, platform_key): - from gateway.run import safe_schedule_threadsafe ctx = self._ctx - # Set up stream consumer for token streaming or interim commentary. - _stream_consumer = None - _stream_delta_cb = None - # streaming TTS consumer is created on the outer event-loop thread before run_sync launches. - # run_sync only reads it via ``streaming_tts_consumer_holder[0]`` for delta callback wiring. - _stts_consumer_ref = ctx.streaming_tts_consumer_holder[0] - _scfg = getattr(getattr(self._runner, 'config', None), 'streaming', None) - if _scfg is None: + stream_consumer = None + stream_delta_cb = None + # The streaming-TTS consumer is created on the outer loop thread before run_sync launches; + # run_sync only reads it via the holder for delta-callback wiring. + stts = ctx.streaming_tts_consumer_holder[0] + scfg = getattr(getattr(self._runner, 'config', None), 'streaming', None) + if scfg is None: from gateway.config import StreamingConfig - _scfg = StreamingConfig() - - # Per-platform streaming gate: display.platforms..streaming can disable streaming - # for specific platforms even when the global streaming config is enabled. - _plat_streaming = ctx.resolve_display_setting( - ctx.user_config, platform_key, "streaming" + scfg = StreamingConfig() + # display.platforms..streaming may disable streaming per platform; None = follow global. + plat_streaming = ctx.resolve_display_setting(ctx.user_config, platform_key, "streaming") + want_stream_deltas = ( + scfg.enabled and scfg.transport != "off" if plat_streaming is None else bool(plat_streaming) ) - # None = no per-platform override → follow global config - _streaming_enabled = ( - _scfg.enabled and _scfg.transport != "off" - if _plat_streaming is None - else bool(_plat_streaming) - ) - _want_stream_deltas = _streaming_enabled - _want_interim_messages = ctx.interim_assistant_messages_enabled - _want_interim_consumer = _want_interim_messages - if _want_stream_deltas or _want_interim_consumer: + want_interim_messages = ctx.interim_assistant_messages_enabled + if want_stream_deltas or want_interim_messages: try: from gateway.stream_consumer import GatewayStreamConsumer - _adapter = self._runner._adapter_for_source(ctx.source) - if _adapter: - _consumer_cfg, _pause_typing_before_finalize = ( - self._runner._build_stream_consumer_config( - ctx.source, _scfg, _adapter, - on_missing_cursor="raise", - ) + adapter = self._runner._adapter_for_source(ctx.source) + if adapter: + consumer_cfg, pause_typing_before_finalize = self._runner._build_stream_consumer_config( + ctx.source, scfg, adapter, on_missing_cursor="raise", ) - _stream_consumer = GatewayStreamConsumer( - adapter=_adapter, + stream_consumer = GatewayStreamConsumer( + adapter=adapter, chat_id=ctx.source.chat_id, - config=_consumer_cfg, + config=consumer_cfg, metadata=ctx._status_thread_metadata, on_new_message=( (lambda: ctx.progress_queue.put(("__reset__",))) - if ctx.progress_queue is not None - else None + if ctx.progress_queue is not None else None ), - on_before_finalize=_pause_typing_before_finalize, + on_before_finalize=pause_typing_before_finalize, initial_reply_to_id=ctx.event_message_id, run_still_current=ctx._run_still_current, ) - if _want_stream_deltas: - def _stream_delta_cb(text: str) -> None: + if want_stream_deltas: + def stream_delta_cb(text: str) -> None: if ctx._run_still_current(): - _stream_consumer.on_delta(text) - # Tee to the streaming-TTS consumer (#60671). - if _stts_consumer_ref is not None: - _stts_consumer_ref.on_delta(text) - ctx.stream_consumer_holder[0] = _stream_consumer - except Exception as _sc_err: - logger.debug("Could not set up stream consumer: %s", _sc_err) - - # Text streaming off but streaming TTS active: install a TTS-only delta callback so the - # consumer still receives LLM deltas for audio synthesis. - if _stream_delta_cb is None and _stts_consumer_ref is not None: - def _stream_delta_cb(text: str) -> None: + stream_consumer.on_delta(text) + if stts is not None: # tee to streaming TTS + stts.on_delta(text) + ctx.stream_consumer_holder[0] = stream_consumer + except Exception as err: + logger.debug("Could not set up stream consumer: %s", err) + # Text streaming off but streaming TTS active: TTS-only delta callback for audio synthesis. + if stream_delta_cb is None and stts is not None: + def stream_delta_cb(text: str) -> None: if ctx._run_still_current(): - _stts_consumer_ref.on_delta(text) + stts.on_delta(text) - def _interim_assistant_cb(text: str, *, already_streamed: bool = False) -> None: + def interim_assistant_cb(text: str, *, already_streamed: bool = False) -> None: if not ctx._run_still_current(): return - display_text = text - if _stream_consumer is not None: + if stream_consumer is not None: if already_streamed: - _stream_consumer.on_segment_break() + stream_consumer.on_segment_break() else: - _stream_consumer.on_commentary(display_text) + stream_consumer.on_commentary(text) return - if already_streamed or not ctx._status_adapter or not str(display_text or "").strip(): + if already_streamed or not ctx._status_adapter or not str(text or "").strip(): return - safe_schedule_threadsafe( - ctx._status_adapter.send( - ctx._status_chat_id, - display_text, - metadata=ctx._status_thread_metadata, - ), - ctx._loop_for_step, - logger=logger, - log_message="interim_assistant_callback scheduling error", - ) - return _stream_consumer, _stream_delta_cb, _interim_assistant_cb, _want_interim_messages + self._send_status_text(text, ctx._status_thread_metadata, "interim_assistant_callback scheduling error") - def _resolve_turn_agent( - self, turn_route, platform_key, combined_ephemeral, max_iterations, reasoning_config, pr, - ): - from gateway.run import _AGENT_PENDING_SENTINEL, _checkpoint_agent_kwargs - ctx = self._ctx - # Per-platform skip_context_files — messaging platforms can opt out of filesystem-heavy - # context-file discovery (SOUL.md, AGENTS.md, .cursorrules) to cut AIAgent build latency. - _platforms_gw_cfg = (ctx.user_config.get("gateway") or {}).get("platforms") or {} + return stream_consumer, stream_delta_cb, interim_assistant_cb, want_interim_messages + + # ── agent resolution (cache reuse vs fresh build) ─────────────────────────────────────── + + @dataclasses.dataclass + class _CachedAgentLookup: + agent: Any = None + reused: bool = False + evicted: Any = None # agent evicted under the lock; released off-lock on a daemon thread + + def _skip_context_files(self, platform_key) -> bool: + """gateway.platforms..skip_context_files: messaging platforms may opt out of + filesystem-heavy context-file discovery (SOUL.md, AGENTS.md, .cursorrules).""" + platforms_cfg = (self._ctx.user_config.get("gateway") or {}).get("platforms") or {} # ``hermes gateway setup`` writes ``gateway.platforms`` as a LIST of enabled platform names, # not a dict; treat any non-dict shape as "no per-platform overrides" rather than crashing. - if not isinstance(_platforms_gw_cfg, dict): - _platforms_gw_cfg = {} - _plat_gw_cfg = _platforms_gw_cfg.get(platform_key) or {} - _skip_context = _plat_gw_cfg.get("skip_context_files") - skip_context_files = bool(_skip_context) if _skip_context is not None else False + if not isinstance(platforms_cfg, dict): + platforms_cfg = {} + skip = (platforms_cfg.get(platform_key) or {}).get("skip_context_files") + return bool(skip) if skip is not None else False - # Agent cache: reuse this session's previous AIAgent to preserve the frozen system prompt - # and tool schemas for prompt cache hits. - _sig = self._runner._agent_config_signature( - turn_route["model"], - turn_route["runtime"], - ctx.enabled_toolsets, - combined_ephemeral, - cache_keys=self._runner._extract_cache_busting_config(ctx.user_config), + def _cached_sid_is_dead(self, cache_lock, cache) -> tuple: + """(peeked cached session_id, is_dead) — checked OUTSIDE the cache lock. + + "cached sid != current sid" normally means an intentional switch (reuse the agent), but the + routing-key self-heal yields the same shape with an agent bound to a DEAD session; reusing it + re-binds the dead sid and loops. + """ + ctx = self._ctx + peek_sid = None + if cache_lock and cache is not None: + with cache_lock: + entry = cache.get(ctx.session_key) + if entry and len(entry) > 3: + peek_sid = entry[3] + dead = False + if peek_sid is not None and ctx.session_id is not None and peek_sid != ctx.session_id: + try: + dead = self._runner.session_store._is_session_ended_in_db(peek_sid) + except Exception: + dead = False + return peek_sid, dead + + def _current_message_count(self): + """Cross-process write guard input: the session's current DB message_count (or None).""" + ctx = self._ctx + if self._runner._session_db is None or not ctx.session_id: + return None + try: + # run_sync is off-loop (executor); sync DB is fine. + row = self._runner._session_db._db.get_session(ctx.session_id) + return row.get("message_count", 0) if row else None + except Exception: + return None + + def _pop_cached_agent_for_eviction(self): + """Evict under the lock but DEFER release: release_clients can block on memory-provider or + socket teardown, stalling the loop while the idle sweeper waits on this lock (blocking + Discord heartbeats). The turn rebuilds a fresh agent, so the caller does a SOFT release + that keeps its terminal sandbox / browser / bg processes — mirrors _evict_cached_agent.""" + from gateway.run import _AGENT_PENDING_SENTINEL + evicted = self._runner._agent_cache.pop(self._ctx.session_key, None) + agent = evicted[0] if isinstance(evicted, tuple) and evicted else None + return agent if agent and agent is not _AGENT_PENDING_SENTINEL else None + + def _lookup_cached_agent(self, sig, cache_lock, cache, max_iterations, peek_sid, dead, msg_count): + ctx = self._ctx + out = self._CachedAgentLookup() + if not (cache_lock and cache is not None): + return out + with cache_lock: + cached = cache.get(ctx.session_key) + if not (cached and cached[1] == sig): + return out + # cached[2] = message_count at cache time (stale when a second process appended rows); + # cached[3] = the session_id the snapshot was taken for. + cached_mc = cached[2] if len(cached) > 2 else None + cached_sid = cached[3] if len(cached) > 3 else None + # Same session_key, other conversation: the counts track DIFFERENT DB rows, so the + # comparison is meaningless — REUSE rather than bust the prompt cache on every switch. + sid_mismatch = cached_sid is not None and ctx.session_id is not None and cached_sid != ctx.session_id + # Re-validate the outside-lock dead-session peek against the tuple read under THIS lock: + # a stale "dead" verdict must never be applied to a different (possibly live) agent. + if sid_mismatch and dead and cached_sid == peek_sid: + logger.info( + "Agent cache invalidated for session %s: " + "cached agent's session_id %s is ended in " + "state.db (stale self-heal artifact, " + "#54878 x #54947) — discarding instead of " + "reusing across the routing recovery", ctx.session_key, cached_sid, + ) + out.evicted = self._pop_cached_agent_for_eviction() + elif not sid_mismatch and cached_mc is not None and msg_count is not None and msg_count != cached_mc: + logger.info( + "Agent cache invalidated for session %s: " + "message_count changed (%s -> %s), " + "possible cross-process write", ctx.session_key, cached_mc, msg_count, + ) + out.evicted = self._pop_cached_agent_for_eviction() + else: + out.agent = cached[0] + # Refresh LRU order so cap enforcement evicts truly-oldest entries. + if hasattr(cache, "move_to_end"): + with suppress(KeyError): + cache.move_to_end(ctx.session_key) + self._runner._init_cached_agent_for_turn(out.agent, ctx._interrupt_depth) + # Cached agent may have been created with old config. + out.agent.max_iterations = max_iterations + logger.debug("Reusing cached agent for session %s", ctx.session_key) + out.reused = True + return out + + def _release_evicted_agent(self, agent) -> None: + """Off-lock soft release on a daemon thread so teardown never blocks the gateway loop.""" + try: + threading.Thread( + target=self._runner._release_evicted_agent_soft, args=(agent,), daemon=True, + name=f"agent-xproc-evict-{str(self._ctx.session_key)[:24]}", + ).start() + except Exception: + # Interpreter shutdown or thread-spawn failure — release inline as a best-effort fallback. + with suppress(Exception): + self._runner._release_evicted_agent_soft(agent) + + def _build_fresh_agent(self, turn_route, platform_key, combined_ephemeral, max_iterations, + reasoning_config, pr, skip_context_files): + from gateway.run import _checkpoint_agent_kwargs + ctx = self._ctx + runner = self._runner + return ctx.AIAgent( + model=turn_route["model"], + **turn_route["runtime"], + **_checkpoint_agent_kwargs(ctx.user_config), + max_iterations=max_iterations, + quiet_mode=True, + verbose_logging=False, + enabled_toolsets=ctx.enabled_toolsets, + disabled_toolsets=ctx.disabled_toolsets, + ephemeral_system_prompt=combined_ephemeral or None, + prefill_messages=runner._prefill_messages or None, + reasoning_config=reasoning_config, + service_tier=runner._service_tier, + request_overrides=turn_route.get("request_overrides"), + providers_allowed=pr.get("only"), + providers_ignored=pr.get("ignore"), + providers_order=pr.get("order"), + provider_sort=pr.get("sort"), + provider_require_parameters=pr.get("require_parameters", False), + provider_data_collection=pr.get("data_collection"), + session_id=ctx.session_id, + platform=platform_key, + user_id=ctx.source.user_id, + user_id_alt=ctx.source.user_id_alt, + user_name=ctx.source.user_name, + chat_id=ctx.source.chat_id, + chat_name=ctx.source.chat_name, + chat_type=ctx.source.chat_type, + thread_id=ctx.source.thread_id, + gateway_session_key=ctx.session_key, + session_db=getattr(runner._session_db, "_db", runner._session_db), + # Reload from disk — do not reuse the startup snapshot. + fallback_model=self._runner._refresh_fallback_model(), + skip_context_files=skip_context_files, + # Keep the persona even with minimal context: soul identity is one small file. + load_soul_identity=True, + ) + + def _resolve_turn_agent(self, turn_route, platform_key, combined_ephemeral, max_iterations, reasoning_config, pr): + """Reuse this session's cached AIAgent (frozen system prompt + tool schemas → prompt cache + hits) or build a fresh one. Returns (agent, reused_cached_agent).""" + ctx = self._ctx + runner = self._runner + skip_context_files = self._skip_context_files(platform_key) + sig = runner._agent_config_signature( + turn_route["model"], turn_route["runtime"], ctx.enabled_toolsets, combined_ephemeral, + cache_keys=runner._extract_cache_busting_config(ctx.user_config), user_id=getattr(ctx.source, "user_id", None), user_id_alt=getattr(ctx.source, "user_id_alt", None), skip_context_files=skip_context_files, ) - agent = None - reused_cached_agent = False - _cache_lock = getattr(self._runner, "_agent_cache_lock", None) - _cache = getattr(self._runner, "_agent_cache", None) - - # Peek at the cached entry's snapshot session_id so we can check, OUTSIDE the cache lock, - # whether it is a DEAD session in state.db. "cached sid != current sid" normally means an - # intentional switch (reuse the agent), but the routing-key self-heal yields the same shape - # with an agent bound to a DEAD session; reusing it re-binds the dead sid and loops. - _peek_cached_sid = None - if _cache_lock and _cache is not None: - with _cache_lock: - _peek_entry = _cache.get(ctx.session_key) - if _peek_entry and len(_peek_entry) > 3: - _peek_cached_sid = _peek_entry[3] - _cached_sid_is_dead = False - if ( - _peek_cached_sid is not None - and ctx.session_id is not None - and _peek_cached_sid != ctx.session_id - ): - try: - _cached_sid_is_dead = self._runner.session_store._is_session_ended_in_db( - _peek_cached_sid - ) - except Exception: - _cached_sid_is_dead = False - - # Cross-process write guard: another process (e.g. hermes dashboard) appending to the same - # SessionDB session makes the cached agent's transcript stale. On message_count mismatch vs - # the count recorded at cache time, invalidate so a fresh agent re-reads from disk. - _current_msg_count = None - if self._runner._session_db is not None and ctx.session_id: - try: - # run_sync is off-loop (executor); sync DB is fine. - _sess_row = self._runner._session_db._db.get_session(ctx.session_id) - if _sess_row: - _current_msg_count = _sess_row.get("message_count", 0) - except Exception: - pass - - _xproc_evicted_agent = None - if _cache_lock and _cache is not None: - with _cache_lock: - cached = _cache.get(ctx.session_key) - if cached and cached[1] == _sig: - # cached[2] is the message_count at cache time; stale when a second process - # appended rows. cached[3] (when present) is the session_id the snapshot was - # taken for — used to skip the guard when the active session_id differs. - _cached_mc = cached[2] if len(cached) > 2 else None - _cached_sid = cached[3] if len(cached) > 3 else None - # Snapshot from a different session_id (same session_key, other conversation): the - # counts track DIFFERENT DB rows, so the comparison is meaningless. REUSE the cached - # agent rather than rebuild and bust the prompt cache on every session switch. - _session_id_mismatch = ( - _cached_sid is not None - and ctx.session_id is not None - and _cached_sid != ctx.session_id - ) - # Re-validate the OUTSIDE-lock dead-session peek against the tuple read under THIS - # lock: the entry may have been replaced between peek and acquisition, and a stale - # "dead" verdict must never be applied to a different (possibly live) cached agent. - _stale_dead_sid_reuse = ( - _session_id_mismatch - and _cached_sid_is_dead - and _cached_sid == _peek_cached_sid - ) - if _stale_dead_sid_reuse: - # The routing key was just self-healed away from a session state.db marked - # ended, but this cached AIAgent still belongs to that DEAD session_id. - # Reusing it would re-bind the dead sid and undo the self-heal; rebuild fresh. - logger.info( - "Agent cache invalidated for session %s: " - "cached agent's session_id %s is ended in " - "state.db (stale self-heal artifact, " - "#54878 x #54947) — discarding instead of " - "reusing across the routing recovery", - ctx.session_key, _cached_sid, - ) - evicted = self._runner._agent_cache.pop(ctx.session_key, None) - _ev_agent = evicted[0] if isinstance(evicted, tuple) and evicted else None - if _ev_agent and _ev_agent is not _AGENT_PENDING_SENTINEL: - # Same deferred-cleanup rationale as the cross-process branch below: don't - # block the event loop / cache lock on memory-provider or socket teardown. - _xproc_evicted_agent = _ev_agent - elif ( - not _session_id_mismatch - and _cached_mc is not None - and _current_msg_count is not None - and _current_msg_count != _cached_mc - ): - # Cross-process write detected — discard stale - # agent so it rebuilds from fresh DB transcript. - logger.info( - "Agent cache invalidated for session %s: " - "message_count changed (%s -> %s), " - "possible cross-process write", - ctx.session_key, _cached_mc, _current_msg_count, - ) - evicted = self._runner._agent_cache.pop(ctx.session_key, None) - _ev_agent = evicted[0] if isinstance(evicted, tuple) and evicted else None - if _ev_agent and _ev_agent is not _AGENT_PENDING_SENTINEL: - # Defer cleanup until AFTER the lock is released: release_clients can - # block on memory-provider/socket teardown, stalling the event loop while - # the idle sweeper waits on this lock (blocking Discord heartbeats). The - # session rebuilds a fresh agent below, so use the SOFT release that keeps - # its terminal sandbox / browser / bg processes for the new agent to - # inherit — mirrors _evict_cached_agent / idle-sweep. - _xproc_evicted_agent = _ev_agent - else: - agent = cached[0] - # Refresh LRU order so the cap enforcement evicts - # truly-oldest entries, not the one we just used. - if hasattr(_cache, "move_to_end"): - with suppress(KeyError): - _cache.move_to_end(ctx.session_key) - self._runner._init_cached_agent_for_turn(agent, ctx._interrupt_depth) - # Refresh agent max_iterations from current config - # (cached agent may have been created with old config) - agent.max_iterations = max_iterations - logger.debug("Reusing cached agent for session %s", ctx.session_key) - reused_cached_agent = True - + cache_lock = getattr(runner, "_agent_cache_lock", None) + cache = getattr(runner, "_agent_cache", None) + peek_sid, dead = self._cached_sid_is_dead(cache_lock, cache) + msg_count = self._current_message_count() + found = self._lookup_cached_agent(sig, cache_lock, cache, max_iterations, peek_sid, dead, msg_count) + agent = found.agent # Lock released — refresh the reused agent's fallback chain from disk OUTSIDE the cache lock # (disk I/O under the lock stalls the idle-sweep watcher and Discord heartbeats). A chain # configured after caching must reach the next turn; per-session serialization keeps it safe. - if reused_cached_agent and agent is not None: - self._runner._apply_fallback_chain_to_agent( - agent, self._runner._refresh_fallback_model(), - ) - - # Lock released — schedule cleanup of any cross-process-evicted agent on a daemon thread so - # memory-provider/socket teardown never blocks the gateway loop or the expiry watcher's lock. - if _xproc_evicted_agent is not None: - try: - threading.Thread( - target=self._runner._release_evicted_agent_soft, - args=(_xproc_evicted_agent,), - daemon=True, - name=f"agent-xproc-evict-{str(ctx.session_key)[:24]}", - ).start() - except Exception: - # Interpreter shutdown or thread-spawn failure — release - # inline as a best-effort fallback. - with suppress(Exception): - self._runner._release_evicted_agent_soft(_xproc_evicted_agent) - + if found.reused and agent is not None: + self._runner._apply_fallback_chain_to_agent(agent, runner._refresh_fallback_model()) + if found.evicted is not None: + self._release_evicted_agent(found.evicted) if agent is None: - # Config changed or first message — create fresh agent - agent = ctx.AIAgent( - model=turn_route["model"], - **turn_route["runtime"], - **_checkpoint_agent_kwargs(ctx.user_config), - max_iterations=max_iterations, - quiet_mode=True, - verbose_logging=False, - enabled_toolsets=ctx.enabled_toolsets, - disabled_toolsets=ctx.disabled_toolsets, - ephemeral_system_prompt=combined_ephemeral or None, - prefill_messages=self._runner._prefill_messages or None, - reasoning_config=reasoning_config, - service_tier=self._runner._service_tier, - request_overrides=turn_route.get("request_overrides"), - providers_allowed=pr.get("only"), - providers_ignored=pr.get("ignore"), - providers_order=pr.get("order"), - provider_sort=pr.get("sort"), - provider_require_parameters=pr.get("require_parameters", False), - provider_data_collection=pr.get("data_collection"), - session_id=ctx.session_id, - platform=platform_key, - user_id=ctx.source.user_id, - user_id_alt=ctx.source.user_id_alt, - user_name=ctx.source.user_name, - chat_id=ctx.source.chat_id, - chat_name=ctx.source.chat_name, - chat_type=ctx.source.chat_type, - thread_id=ctx.source.thread_id, - gateway_session_key=ctx.session_key, - session_db=getattr(self._runner._session_db, "_db", self._runner._session_db), - # Reload from disk — do not reuse the startup snapshot (#60955). - fallback_model=self._runner._refresh_fallback_model(), - skip_context_files=skip_context_files, - # Keep the persona even with minimal context: soul identity is - # a single small file, not part of the expensive walk. - load_soul_identity=True, + agent = self._build_fresh_agent( + turn_route, platform_key, combined_ephemeral, max_iterations, reasoning_config, pr, + skip_context_files, ) - if _cache_lock and _cache is not None: - with _cache_lock: - # Record the snapshot's session_id with message_count so the cross-process guard can - # skip the meaningless count comparison if the active session_id later switches. - _cache[ctx.session_key] = ( - agent, _sig, _current_msg_count, ctx.session_id, - ) - self._runner._enforce_agent_cache_cap() - logger.debug("Created new agent for session %s (sig=%s)", ctx.session_key, _sig) - return agent, reused_cached_agent + if cache_lock and cache is not None: + with cache_lock: + # Record the snapshot's session_id with message_count so the cross-process guard + # can skip the meaningless count comparison if the active session_id switches. + cache[ctx.session_key] = (agent, sig, msg_count, ctx.session_id) + runner._enforce_agent_cache_cap() + logger.debug("Created new agent for session %s (sig=%s)", ctx.session_key, sig) + return agent, found.reused - def _wire_turn_agent_callbacks( - self, agent, turn_route, reasoning_config, - _stream_delta_cb, _interim_assistant_cb, _want_interim_messages, - ): - from gateway.run import ( - _interim_metadata, - _non_conversational_metadata, - render_notice_line, - safe_schedule_threadsafe, - ) - ctx = self._ctx - # Per-message state — callbacks and reasoning config change every turn, so they aren't baked - # into the cached agent. The progress callback is ALWAYS attached (never gated to None): its - # body gates each event class, and subagent-failure notices must fire even with - # tool_progress/thinking off — a None gate made dead subagents vanish silently. - agent.tool_progress_callback = ctx.progress_callback - # Compose ID-bearing lifecycle consumers: Discord's one-time voice ack and Slack's task cards - # both ride the authoritative start callback, so neither infers identity from tool names. - _combined_start_cb = ctx.native_tool_start_callback or ctx.voice_ack_callback - agent.tool_start_callback = ( - _combined_start_cb - if ( - ctx._voice_ack_guild[0] is not None - or ctx._native_slack_task_cards + # ── per-turn agent wiring ─────────────────────────────────────────────────────────────── + + def _notice_callback_sync(self, notice) -> None: + """Credits / out-of-band notices (usage bands, depletion, restored) fire from the agent's + sync worker thread; hop onto the gateway loop. Fired-once latch lives on the cached agent.""" + from gateway.run import render_notice_line + if not self._status_live(): + return + try: + line = render_notice_line(notice) + except Exception: + logger.debug("render_notice_line failed", exc_info=True) + return + if line: + self._schedule( + self._runner._deliver_platform_notice(self._ctx.source, line), + "notice_callback delivery scheduling error", ) - else None + + def _make_bg_review_callbacks(self): + """(send, release): background-review messages ("💾 Memory updated") are held until the + adapter's post-delivery hook releases them after the main response lands.""" + from gateway.run import _interim_metadata, _non_conversational_metadata + ctx = self._ctx + release_evt = threading.Event() + pending: list[str] = [] + pending_lock = threading.Lock() + + def deliver(message: str) -> None: + if not self._status_live(): + return + self._send_status_text( + message, + _interim_metadata(_non_conversational_metadata(ctx._status_thread_metadata, platform=ctx.source.platform)), + "background_review_callback scheduling error", + ) + + def release() -> None: + release_evt.set() + with pending_lock: + queued = list(pending) + pending.clear() + for message in queued: + deliver(message) + + def send(message: str) -> None: + if not self._status_live(): + return + if not release_evt.is_set(): + with pending_lock: + if not release_evt.is_set(): + pending.append(message) + return + deliver(message) + + return send, release + + @staticmethod + def _merge_turn_request_overrides(agent, turn_route) -> None: + """Merge, never overwrite: init-time request overrides (e.g. a custom provider's extra_body) + must survive every reused-agent turn. Drop only the PREVIOUS turn's routing overrides before + layering this turn's, so stale per-turn values never linger.""" + overrides = dict(getattr(agent, "request_overrides", {}) or {}) + for key, value in (dict(getattr(agent, "_gateway_turn_request_overrides", {}) or {})).items(): + if overrides.get(key) == value: + overrides.pop(key, None) + turn_overrides = dict(turn_route.get("request_overrides") or {}) + overrides.update(turn_overrides) + agent.request_overrides = overrides + agent._gateway_turn_request_overrides = turn_overrides + + def _wire_turn_agent_callbacks(self, agent, turn_route, reasoning_config, + stream_delta_cb, interim_assistant_cb, want_interim_messages): + """Per-message state — callbacks and reasoning config change every turn, so they aren't + baked into the cached agent.""" + ctx = self._ctx + runner = self._runner + # ALWAYS attached (never gated to None): its body gates each event class, and subagent- + # failure notices must fire even with tool_progress/thinking off. + agent.tool_progress_callback = ctx.progress_callback + # Discord's one-time voice ack and Slack's task cards both ride the authoritative start + # callback, so neither infers identity from tool names. + agent.tool_start_callback = ( + (ctx.native_tool_start_callback or ctx.voice_ack_callback) + if (ctx._voice_ack_guild[0] is not None or ctx._native_slack_task_cards) else None ) agent.tool_complete_callback = ( ctx.native_tool_complete_callback - if ctx._native_slack_task_cards - and ctx.native_tool_complete_callback is not None - else None + if ctx._native_slack_task_cards and ctx.native_tool_complete_callback is not None else None ) agent.step_callback = ctx._step_callback_sync if ctx._hooks_ref.loaded_hooks else None - agent.stream_delta_callback = _stream_delta_cb - agent.interim_assistant_callback = _interim_assistant_cb if _want_interim_messages else None + agent.stream_delta_callback = stream_delta_cb + agent.interim_assistant_callback = interim_assistant_cb if want_interim_messages else None agent.status_callback = ctx._status_callback_sync - # Credits / out-of-band notices (usage bands, depletion, restored) fire from the agent's sync - # worker thread, so hop onto the gateway loop via safe_schedule_threadsafe. Fired-once latch - # lives on the cached agent (no per-turn re-nag); clear is a no-op — sends can't be retracted. - def _notice_callback_sync(notice) -> None: - if not ctx._status_adapter or not ctx._run_still_current(): - return - try: - line = render_notice_line(notice) - except Exception: - logger.debug("render_notice_line failed", exc_info=True) - return - if not line: - return - safe_schedule_threadsafe( - self._runner._deliver_platform_notice(ctx.source, line), - ctx._loop_for_step, - logger=logger, - log_message="notice_callback delivery scheduling error", - ) - - agent.notice_callback = _notice_callback_sync - agent.notice_clear_callback = None + agent.notice_callback = self._notice_callback_sync + agent.notice_clear_callback = None # sends can't be retracted agent.event_callback = ctx._event_callback_sync agent.reasoning_config = reasoning_config - agent.service_tier = self._runner._service_tier - # Merge, never overwrite: init-time request overrides (e.g. a custom provider's extra_body - # merged at agent construction) must survive every reused-agent turn. Drop only the PREVIOUS - # turn's routing overrides before layering this turn's, so stale per-turn values never linger. - request_overrides = dict(getattr(agent, "request_overrides", {}) or {}) - previous_turn_overrides = dict( - getattr(agent, "_gateway_turn_request_overrides", {}) or {} - ) - for key, value in previous_turn_overrides.items(): - if request_overrides.get(key) == value: - request_overrides.pop(key, None) - turn_request_overrides = dict(turn_route.get("request_overrides") or {}) - request_overrides.update(turn_request_overrides) - agent.request_overrides = request_overrides - agent._gateway_turn_request_overrides = turn_request_overrides + agent.service_tier = runner._service_tier + self._merge_turn_request_overrides(agent, turn_route) # Must-deliver notes for THIS turn ride the current user message (api_content sidecar), never - # the system prompt: staged by _handle_message_with_agent (auto-reset, first-contact intro, - # voice-channel change). Assigned unconditionally so a reused agent never replays a stale note. - agent._gateway_turn_context_notes = "\n\n".join( - self._runner._consume_pending_turn_sidecar_notes(ctx.session_key) - ) - - _bg_review_release = threading.Event() - _bg_review_pending: list[str] = [] - _bg_review_pending_lock = threading.Lock() - - def _deliver_bg_review_message(message: str) -> None: - if not ctx._status_adapter or not ctx._run_still_current(): - return - safe_schedule_threadsafe( - ctx._status_adapter.send( - ctx._status_chat_id, - message, - metadata=_interim_metadata(_non_conversational_metadata(ctx._status_thread_metadata, platform=ctx.source.platform)), - ), - ctx._loop_for_step, - logger=logger, - log_message="background_review_callback scheduling error", - ) - - def _release_bg_review_messages() -> None: - _bg_review_release.set() - with _bg_review_pending_lock: - pending = list(_bg_review_pending) - _bg_review_pending.clear() - for queued in pending: - _deliver_bg_review_message(queued) - - # Background review delivery — send "💾 Memory updated" etc. to user - def _bg_review_send(message: str) -> None: - if not ctx._status_adapter or not ctx._run_still_current(): - return - if not _bg_review_release.is_set(): - with _bg_review_pending_lock: - if not _bg_review_release.is_set(): - _bg_review_pending.append(message) - return - _deliver_bg_review_message(message) - - agent.background_review_callback = _bg_review_send - # Register the release hook on the adapter so base.py's finally - # block can fire it after delivering the main response. + # the system prompt. Assigned unconditionally so a reused agent never replays a stale note. + agent._gateway_turn_context_notes = "\n\n".join(runner._consume_pending_turn_sidecar_notes(ctx.session_key)) + bg_send, bg_release = self._make_bg_review_callbacks() + agent.background_review_callback = bg_send + # Register the release hook on the adapter so base.py's finally block fires it after the + # main response is delivered. if ctx._status_adapter and ctx.session_key: if getattr(type(ctx._status_adapter), "register_post_delivery_callback", None) is not None: ctx._status_adapter.register_post_delivery_callback( - ctx.session_key, - _release_bg_review_messages, - generation=ctx.run_generation, + ctx.session_key, bg_release, generation=ctx.run_generation, ) else: - _pdc = getattr(ctx._status_adapter, "_post_delivery_callbacks", None) - if _pdc is not None: - _pdc[ctx.session_key] = _release_bg_review_messages - # Memory update notifications in chat. Config: display.memory_notifications - # off — no chat notification (still logged to stdout) - # on — generic "💾 Memory updated" (default) - # verbose — content preview: "💾 Memory ➕ Hermes Repo..." - _mem_notif = ctx.user_config.get("display", {}).get("memory_notifications") - if isinstance(_mem_notif, bool): - _mem_notif = "on" if _mem_notif else "off" - agent.memory_notifications = str(_mem_notif).lower() if _mem_notif else "on" - + pdc = getattr(ctx._status_adapter, "_post_delivery_callbacks", None) + if pdc is not None: + pdc[ctx.session_key] = bg_release + # display.memory_notifications: off | on (generic "💾 Memory updated", default) | verbose. + mem_notif = ctx.user_config.get("display", {}).get("memory_notifications") + if isinstance(mem_notif, bool): + mem_notif = "on" if mem_notif else "off" + agent.memory_notifications = str(mem_notif).lower() if mem_notif else "on" agent.clarify_callback = self._clarify_callback_sync - - # Show assistant thinking between tool calls — independent of tool_progress mode. Mattermost - # needs an explicit per-platform opt-in so global scratch-text doesn't leak into threads. + # Thinking between tool calls is independent of tool_progress mode (Mattermost opts in + # per platform so global scratch-text doesn't leak into threads). agent.thinking_progress = ctx._thinking_enabled - # Store agent reference for interrupt support - ctx.agent_holder[0] = agent - # Wire the platform thread-rename lane onto the agent: the titler fires from the turn prologue, - # not after the response, so titles are pushed the moment they land. + ctx.agent_holder[0] = agent # interrupt support + # The titler fires from the turn prologue, so attach the rename lane before the run. self._attach_session_title_callback(agent, ctx) - # Publish turn ownership for explicit /stop, /new, disconnect, and shutdown interrupts. - # Older session processes are outside this baseline and remain alive. + # Publish turn ownership for /stop, /new, disconnect and shutdown interrupts; older session + # processes are outside this baseline and remain alive. agent._gateway_turn_process_task_id = ctx.process_task_id agent._gateway_turn_process_baseline = ctx.process_baseline - # Capture the full tool definitions for transcript logging - ctx.tools_holder[0] = agent.tools if hasattr(agent, 'tools') else None + ctx.tools_holder[0] = agent.tools if hasattr(agent, 'tools') else None # transcript logging - # ------------------------------------------------------------------ - # Shared native-stream boundary close: for native-streaming platforms (e.g. WeCom), an - # interrupting interaction (approval or clarify prompt) must finalize the current stream - # and disable native streaming first, or post-interaction output keeps updating the OLD - # bubble above the prompt. Runs on the agent thread; the consumer serializes via its queue. - def _close_native_stream_boundary( - self, _reason: str, _placeholder: str | None = None, _reopen: bool = False, - ) -> bool: - ctx = self._ctx - _sc = ctx.stream_consumer_holder[0] if ctx.stream_consumer_holder else None - if not (_sc and getattr(_sc, "_use_native_streaming", False)): + # ── blocking prompts from the agent thread (approval / clarify) ───────────────────────── + + def _close_native_stream_boundary(self, reason: str, placeholder: str | None = None, reopen: bool = False) -> bool: + """Native-streaming platforms (e.g. WeCom): an interrupting interaction (approval or clarify + prompt) must finalize the current stream first, or post-interaction output keeps updating the + OLD bubble above the prompt. Runs on the agent thread; the consumer serializes via its queue.""" + sc = self._stream_consumer() + if not (sc and getattr(sc, "_use_native_streaming", False)): return True - _cancelled_flag = None + cancelled_flag = None try: - _boundary_result = _sc.close_for_approval_prompt( - _placeholder, reason=_reason, reopen=_reopen, - ) + boundary = sc.close_for_approval_prompt(placeholder, reason=reason, reopen=reopen) # Returns (future, cancelled_flag) or just a future. - if isinstance(_boundary_result, tuple): - _boundary_future, _cancelled_flag = _boundary_result - else: - _boundary_future = _boundary_result - if hasattr(_boundary_future, "result"): - _ok = _boundary_future.result(timeout=10) - if not _ok: + if isinstance(boundary, tuple): + boundary, cancelled_flag = boundary + if hasattr(boundary, "result"): + ok = boundary.result(timeout=10) + if not ok: logger.warning( "%s boundary failed to close stream properly — " - "prompt may still appear in typing bubble", _reason, + "prompt may still appear in typing bubble", reason, ) - return bool(_ok) + return bool(ok) return True - except (TimeoutError, Exception) as _boundary_err: - if _cancelled_flag is not None: - _cancelled_flag["cancelled"] = True - logger.warning( - "%s boundary timed out or failed: %s", _reason, _boundary_err, - ) + except (TimeoutError, Exception) as err: + if cancelled_flag is not None: + cancelled_flag["cancelled"] = True + logger.warning("%s boundary timed out or failed: %s", reason, err) return False - # ------------------------------------------------------------------ - # Clarify callback: present a clarify prompt and block on a response. Runs on the agent's - # worker thread (clarify_tool's synchronous contract): schedules the adapter's send_clarify - # on the gateway loop, then blocks on the primitive's threading.Event with a timeout. - # Returns the response string, or a sentinel explaining no response arrived. - # ------------------------------------------------------------------ def _clarify_callback_sync(self, question: str, choices, multi_select: bool = False) -> str: - from gateway.run import _clarify_send_then_wait, safe_schedule_threadsafe - ctx = self._ctx - from tools import clarify_gateway as _clarify_mod - import uuid as _uuid + """Present a clarify prompt and block on a response (clarify_tool's synchronous contract). + Schedules the adapter's send_clarify on the gateway loop, then blocks on the primitive's + threading.Event with a timeout. Returns the response string, or a sentinel when none arrived. + """ + from gateway.run import _clarify_send_then_wait + from tools import clarify_gateway as clarify_mod + import uuid + ctx = self._ctx if not ctx._status_adapter: return "" - - clarify_id = _uuid.uuid4().hex[:10] - _clarify_mod.register( - clarify_id=clarify_id, - session_key=ctx.session_key or "", - question=question, - choices=list(choices) if choices else None, - multi_select=bool(multi_select), + session_key = ctx.session_key or "" + clarify_id = uuid.uuid4().hex[:10] + clarify_mod.register( + clarify_id=clarify_id, session_key=session_key, question=question, + choices=list(choices) if choices else None, multi_select=bool(multi_select), ) - - # WeCom native streaming: finalize the current stream before the clarify prompt so the - # post-answer output opens a fresh bubble below the question ("气泡割裂" otherwise). Unlike - # approval, clarify passes reopen=True so the continuation re-opens a native stream; if - # the re-seed fails the consumer degrades to send() automatically. - self._close_native_stream_boundary( - "Clarify", "💬 等待你的选择...", _reopen=True, - ) - - # Pause typing — as with approval, a "thinking..." status must not obscure the prompt or - # block an "Other" reply on platforms that disable input while typing (Slack Assistant). + # Unlike approval, clarify passes reopen=True so the continuation re-opens a native stream + # below the question; if the re-seed fails the consumer degrades to send() automatically. + self._close_native_stream_boundary("Clarify", "💬 等待你的选择...", reopen=True) + # Pause typing: a "thinking..." status must not obscure the prompt or block an "Other" reply + # on platforms that disable input while typing (Slack Assistant). with suppress(Exception): ctx._status_adapter.pause_typing_for_chat(ctx._status_chat_id) - - # Ordering barrier: flush buffered assistant prose to the platform BEFORE sending the - # poll, which goes out on a separate agent-thread-blocking path and would otherwise - # render ABOVE its own explanation. Best-effort + short timeout so the agent thread - # never hangs if the consumer task isn't running. + # Ordering barrier: flush buffered assistant prose BEFORE the poll, which goes out on a + # separate agent-thread-blocking path and would otherwise render ABOVE its own explanation. + # Best-effort + short timeout so the agent thread never hangs if the consumer isn't running. try: - _sc = ctx.stream_consumer_holder[0] if ctx.stream_consumer_holder else None - _flush = getattr(_sc, "flush_pending_sync", None) - if callable(_flush): - _flush(timeout=3.0) + flush = getattr(self._stream_consumer(), "flush_pending_sync", None) + if callable(flush): + flush(timeout=3.0) except Exception: - logger.debug( - "Stream-consumer flush before clarify prompt failed", - exc_info=True, - ) - - fut = safe_schedule_threadsafe( + logger.debug("Stream-consumer flush before clarify prompt failed", exc_info=True) + fut = self._schedule( ctx._status_adapter.send_clarify( - chat_id=ctx._status_chat_id, - question=question, - choices=list(choices) if choices else None, - clarify_id=clarify_id, - session_key=ctx.session_key or "", - metadata=ctx._status_thread_metadata, + chat_id=ctx._status_chat_id, question=question, + choices=list(choices) if choices else None, clarify_id=clarify_id, + session_key=session_key, metadata=ctx._status_thread_metadata, ), - ctx._loop_for_step, - logger=logger, - log_message="Clarify send failed to schedule", + "Clarify send failed to schedule", ) # Boundary rule (see _approval_send_outcome): a send timeout is AMBIGUOUS — the card may # have posted with a late ack. Only a definitive failure tears down the registration; # ambiguous falls through to the bounded wait so a late reply resolves. - _clarify_response = _clarify_send_then_wait( - fut, - clarify_id=clarify_id, - session_key=ctx.session_key or "", - clarify_mod=_clarify_mod, - ) + response = _clarify_send_then_wait(fut, clarify_id=clarify_id, session_key=session_key, clarify_mod=clarify_mod) # Only re-arm typing when the user actually answered — the undeliverable sentinel and the # timeout/cancellation strings start with '[' and must pass through untouched. - if not ( - isinstance(_clarify_response, str) - and _clarify_response.startswith("[") - ): - # User answered: reopen typing IMMEDIATELY, not on the LLM's first post-answer token - # (native streaming otherwise re-seeds lazily on the first delta: ~48s of dead air). - # request_reopen_seed is a no-op outside the reopen-pending native state; always safe. - _sc_reopen = ctx.stream_consumer_holder[0] if ctx.stream_consumer_holder else None - if _sc_reopen is not None: + if not (isinstance(response, str) and response.startswith("[")): + # Reopen typing IMMEDIATELY, not on the LLM's first post-answer token (native streaming + # otherwise re-seeds lazily on the first delta: ~48s of dead air). request_reopen_seed is + # a no-op outside the reopen-pending native state. + sc = self._stream_consumer() + if sc is not None: try: - _sc_reopen.request_reopen_seed() + sc.request_reopen_seed() except Exception: - logger.debug( - "request_reopen_seed after clarify answer failed", - exc_info=True, - ) + logger.debug("request_reopen_seed after clarify answer failed", exc_info=True) try: ctx._status_adapter.resume_typing_for_chat(ctx._status_chat_id) except Exception: - logger.debug( - "resume_typing_for_chat after clarify answer failed", - exc_info=True, - ) - return _clarify_response - - def _load_turn_history(self, agent, reused_cached_agent): - from gateway.run import ( - _build_gateway_agent_history, - _collect_history_media_paths, - _message_timestamps_enabled, - _select_cached_agent_history, - ) - ctx = self._ctx - # Convert history to agent format. Transcript path: {role, content, timestamp} dicts — strip - # timestamps. Interrupt path (agent result["messages"]): full agent messages with - # tool_calls/tool_call_id/reasoning — pass through intact so the API sees valid assistant→tool - # sequences (dropping tool_calls causes 500s). Telegram observed group context: observed=True - # rows are withheld from replayable history and attached to the current addressed message as - # API-only context, so persisted history stores only the real addressed user turn. - agent_history, observed_group_context = _build_gateway_agent_history( - ctx.history, - channel_prompt=ctx.channel_prompt, - inject_timestamps=_message_timestamps_enabled(ctx.user_config), - ) - - # FTS write-corruption guard: if persistence failed silently via corrupt FTS triggers, the - # reloaded transcript is stale/empty while the SAME cached agent still holds the full live - # conversation in `_session_messages`; replacing it causes same-session amnesia. Only for - # a reused agent bound to this exact session_id. - if reused_cached_agent and getattr(agent, "session_id", None) == ctx.session_id: - _selected = _select_cached_agent_history( - agent_history, getattr(agent, "_session_messages", None) - ) - if _selected is not agent_history: - logger.warning( - "Persisted transcript lagged live cached history for " - "session %s (disk=%d, memory=%d); preserving live " - "conversation context (possible FTS write corruption)", - ctx.session_key, len(agent_history), len(_selected), - ) - # The live in-memory history bypassed the _build_gateway_agent_history cleanup above — - # re-apply the stale-confirmation expiry so a dangerous confirmation can't slip through. - agent_history = strip_stale_dangerous_confirmations( - _selected, now=time.time() - ) - - # Collect MEDIA paths already in history to exclude them from this turn's extraction. - # Compression-safe: even if the message list shrinks, we know which paths are old. - _history_media_paths: set = _collect_history_media_paths(agent_history) - return agent_history, observed_group_context, _history_media_paths + logger.debug("resume_typing_for_chat after clarify answer failed", exc_info=True) + return response def _approval_notify_sync(self, approval_data: dict) -> None: """Send the approval request to the user from the agent thread. - Uses the adapter's interactive button approvals (e.g. ``send_exec_approval``) when - available, else a plain text message with ``/approve`` instructions. - """ + Uses the adapter's interactive button approvals (``send_exec_approval``) when available, + else a plain text message with ``/approve`` instructions. + """ from gateway.run import ( - _approval_send_outcome, - _format_exec_approval_fallback, - _interim_metadata, + _approval_send_outcome, _format_exec_approval_fallback, _interim_metadata, _redact_approval_command, - safe_schedule_threadsafe, ) ctx = self._ctx - # Pause typing while awaiting approval: Slack's assistant_threads_setStatus disables the - # compose box, so the user can't type /approve while "is thinking..." shows. The approval - # send auto-clears it; pausing stops _keep_typing re-setting it. Resumed in approve/deny. - ctx._status_adapter.pause_typing_for_chat(ctx._status_chat_id) - - # WeCom native streaming: ask the stream consumer to close the current stream before the - # approval prompt — via the consumer's queue, so it serializes with pending deltas. + adapter = ctx._status_adapter + # Slack's assistant_threads_setStatus disables the compose box, so the user can't type + # /approve while "is thinking..." shows. Pausing stops _keep_typing re-setting it; resumed + # in approve/deny. + adapter.pause_typing_for_chat(ctx._status_chat_id) self._close_native_stream_boundary("Approval") - - cmd = approval_data.get("command", "") + # Redact credentials before display: Tirith's findings are already redacted, but the raw + # command string still leaks secrets. Both the button and plain-text paths use this value. + cmd = _redact_approval_command(approval_data.get("command", "")) desc = approval_data.get("description", "dangerous command") - - # Redact credentials from the command before display — Tirith's findings are already - # redacted, but the raw command string still leaks secrets to the chat platform. Done - # here so BOTH the button-based and plain-text fallback paths use the redacted value. - cmd = _redact_approval_command(cmd) - - # Prefer button-based approval when the adapter supports it. Check the *class*, not the - # instance — avoids false positives from MagicMock auto-attribute creation in tests. - if getattr(type(ctx._status_adapter), "send_exec_approval", None) is not None: + flags = { + "allow_permanent": approval_data.get("allow_permanent", True), + "allow_session": approval_data.get("allow_session", True), + "smart_denied": approval_data.get("smart_denied", False), + } + # Check the *class*, not the instance — MagicMock auto-creates attributes in tests. + if getattr(type(adapter), "send_exec_approval", None) is not None: try: - _approval_fut = safe_schedule_threadsafe( - ctx._status_adapter.send_exec_approval( - chat_id=ctx._status_chat_id, - command=cmd, - session_key=ctx.session_key or "", - description=desc, - metadata=ctx._status_thread_metadata, - allow_permanent=approval_data.get("allow_permanent", True), - allow_session=approval_data.get("allow_session", True), - smart_denied=approval_data.get("smart_denied", False), + fut = self._schedule( + adapter.send_exec_approval( + chat_id=ctx._status_chat_id, command=cmd, session_key=ctx.session_key or "", + description=desc, metadata=ctx._status_thread_metadata, **flags, ), - ctx._loop_for_step, - logger=logger, - log_message="send_exec_approval scheduling error", + "send_exec_approval scheduling error", ) - if _approval_fut is None: + if fut is None: raise RuntimeError("send_exec_approval: loop unavailable") - _outcome = _approval_send_outcome(_approval_fut, timeout=15) - if _outcome == "sent": + outcome = _approval_send_outcome(fut, timeout=15) + if outcome == "sent": return - if _outcome == "ambiguous": - # Timeout ≠ failure: the card may have posted with a late ack (slow API or - # backpressure). The prompt registration stays alive so a tap still resolves; - # re-sending made duplicate cards + orphaned "/approve: nothing pending". Skip. + if outcome == "ambiguous": + # Timeout ≠ failure: the card may have posted with a late ack. The prompt + # registration stays alive so a tap still resolves; re-sending made duplicate + # cards + orphaned "/approve: nothing pending". logger.warning( "Button-based approval send timed out — treating " "as possibly-delivered (no re-send; the prompt " "stays armed for a late tap)" ) return - logger.warning( - "Button-based approval failed (send returned error), falling back to text" - ) - except Exception as _e: - logger.warning( - "Button-based approval failed, falling back to text: %s", _e - ) - - # Fallback: plain-text approval prompt with the adapter's typed prefix (e.g. `!approve`) — - # typed "/" is blocked in Slack threads and reserved by Matrix clients. - _p = getattr(ctx._status_adapter, "typed_command_prefix", "/") - msg = _format_exec_approval_fallback( - cmd, - desc, - _p, - allow_permanent=approval_data.get("allow_permanent", True), - allow_session=approval_data.get("allow_session", True), - smart_denied=approval_data.get("smart_denied", False), - ) + logger.warning("Button-based approval failed (send returned error), falling back to text") + except Exception as e: + logger.warning("Button-based approval failed, falling back to text: %s", e) + # Plain-text prompt with the adapter's typed prefix (e.g. `!approve`): typed "/" is blocked + # in Slack threads and reserved by Matrix clients. + msg = _format_exec_approval_fallback(cmd, desc, getattr(adapter, "typed_command_prefix", "/"), **flags) try: - # Mark as approval prompt so WeCom routes through control lane - _approval_metadata = dict(ctx._status_thread_metadata or {}) - _approval_metadata["is_approval_prompt"] = True - - _approval_send_fut = safe_schedule_threadsafe( - ctx._status_adapter.send( - ctx._status_chat_id, - msg, - metadata=_interim_metadata(_approval_metadata), - ), - ctx._loop_for_step, - logger=logger, - log_message="Approval text-send scheduling error", + # Mark as approval prompt so WeCom routes through the control lane. + metadata = dict(ctx._status_thread_metadata or {}) + metadata["is_approval_prompt"] = True + fut = self._schedule( + adapter.send(ctx._status_chat_id, msg, metadata=_interim_metadata(metadata)), + "Approval text-send scheduling error", ) - if _approval_send_fut is not None: - _approval_send_fut.result(timeout=15) - except Exception as _e: - logger.error("Failed to send approval request: %s", _e) + if fut is not None: + fut.result(timeout=15) + except Exception as e: + logger.error("Failed to send approval request: %s", e) - def _prepare_turn_message(self, agent_history): + # ── run_sync phases ───────────────────────────────────────────────────────────────────── + + def _load_turn_history(self, agent, reused_cached_agent): from gateway.run import ( - _auto_continue_freshness_window, - _is_fresh_gateway_interruption, - _last_transcript_timestamp, - _prepare_resume_pending_message, - build_resume_recovery_note, + _build_gateway_agent_history, _collect_history_media_paths, _message_timestamps_enabled, + _select_cached_agent_history, ) ctx = self._ctx - # Keep real user text separate from API-only recovery guidance: if an auto-continue note is - # prepended below, persist the original so stale guidance never replays as user text. - _persist_user_message_override: Optional[Any] = ctx.persist_user_message - _persist_user_timestamp_override: Optional[float] = ctx.persist_user_timestamp - - # Prepend pending model switch note so the model knows about the switch - _pending_notes = getattr(self._runner, '_pending_model_notes', {}) - _msn = _pending_notes.pop(ctx.session_key, None) if ctx.session_key else None - if _msn: - ctx.message = _msn + "\n\n" + ctx.message - - # Auto-continue: history ending with a tool result means the previous turn was cut off - # (restart, crash, SIGTERM) — prepend a system note so the model finishes the pending tool - # results first. Session-level resume_pending (drain-timeout shutdown) uses stronger - # reason-aware wording that subsumes this case. Both gate on the age of ``history[-1]`` (not - # agent_history, which stripped ``timestamp`` off tool rows); rows without one are fresh. - _freshness_window = _auto_continue_freshness_window() - _interruption_is_fresh = _is_fresh_gateway_interruption( - _last_transcript_timestamp(ctx.history), - window_secs=_freshness_window, + # Transcript rows ({role, content, timestamp}) lose timestamps; interrupt-path agent messages + # (tool_calls/tool_call_id/reasoning) pass through intact so the API sees valid assistant→tool + # sequences. Telegram observed=True rows are withheld from replayable history and attached to + # the current addressed message as API-only context. + agent_history, observed_group_context = _build_gateway_agent_history( + ctx.history, channel_prompt=ctx.channel_prompt, + inject_timestamps=_message_timestamps_enabled(ctx.user_config), ) + # FTS write-corruption guard: if persistence failed silently, the reloaded transcript is + # stale/empty while the SAME cached agent still holds the live conversation; replacing it + # causes same-session amnesia. Only for a reused agent bound to this exact session_id. + if reused_cached_agent and getattr(agent, "session_id", None) == ctx.session_id: + selected = _select_cached_agent_history(agent_history, getattr(agent, "_session_messages", None)) + if selected is not agent_history: + logger.warning( + "Persisted transcript lagged live cached history for " + "session %s (disk=%d, memory=%d); preserving live " + "conversation context (possible FTS write corruption)", + ctx.session_key, len(agent_history), len(selected), + ) + # The live history bypassed _build_gateway_agent_history's cleanup — re-apply the + # stale-confirmation expiry so a dangerous confirmation can't slip through. + agent_history = strip_stale_dangerous_confirmations(selected, now=time.time()) + # MEDIA paths already in history are excluded from this turn's extraction (compression-safe). + return agent_history, observed_group_context, _collect_history_media_paths(agent_history) - _resume_entry = None + def _prepend_pending_note(self, attr: str) -> None: + """Consume a one-shot per-session note (model switch, /reload-skills) into the NEXT user + message. Nothing hits the transcript out-of-band, so alternation stays intact.""" + ctx = self._ctx + notes = getattr(self._runner, attr, None) + note = None + if notes and ctx.session_key and ctx.session_key in notes: + note = notes.pop(ctx.session_key, None) + if note: + ctx.message = note + "\n\n" + ctx.message + + def _resume_note_interactive(self) -> bool: + """Interactive platforms report the restore and ask what next; event platforms (webhook, + API server) continue the work — nobody is present to answer.""" + return bool(getattr(self._runner._adapter_for_source(self._ctx.source), "interactive_resume", True)) + + def _prepare_turn_message(self, agent_history): + """Prepend recovery/notice guidance to ``ctx.message``. + + Returns (persist_user_message_override, persist_user_timestamp_override): real user text is + kept separate from API-only recovery guidance so stale guidance never replays as user text. + """ + from gateway.run import ( + _auto_continue_freshness_window, _is_fresh_gateway_interruption, + _last_transcript_timestamp, _prepare_resume_pending_message, build_resume_recovery_note, + ) + ctx = self._ctx + persist_override: Optional[Any] = ctx.persist_user_message + self._prepend_pending_note("_pending_model_notes") + # Auto-continue: history ending with a tool result means the previous turn was cut off + # (restart, crash, SIGTERM). Session-level resume_pending (drain-timeout shutdown) uses + # stronger reason-aware wording that subsumes this case. Both gate on the age of + # ``history[-1]`` (not agent_history, which stripped tool-row timestamps); no stamp = fresh. + window = _auto_continue_freshness_window() + interruption_is_fresh = _is_fresh_gateway_interruption(_last_transcript_timestamp(ctx.history), window_secs=window) + entry = None if ctx.session_key: try: - _resume_entry = self._runner.session_store._entries.get(ctx.session_key) + entry = self._runner.session_store._entries.get(ctx.session_key) except Exception: - _resume_entry = None - - # resume_pending freshness also uses the restart watchdog's ``last_resume_marked_at`` (the - # true interruption stamp): the transcript clock (_interruption_is_fresh) can be hours older - # for an active thread, so gating on it alone drops the recovery note — and the startup - # auto-resume turn has empty text, so the model gets a blank user message. Fresh if EITHER is. - _resume_mark_is_fresh = False - if _resume_entry is not None and getattr(_resume_entry, "resume_pending", False): - _resume_mark_is_fresh = _is_fresh_gateway_interruption( - getattr(_resume_entry, "last_resume_marked_at", None), - window_secs=_freshness_window, - ) - _is_resume_pending = bool( - _resume_entry is not None - and getattr(_resume_entry, "resume_pending", False) - and (_interruption_is_fresh or _resume_mark_is_fresh) + entry = None + resume_pending = entry is not None and getattr(entry, "resume_pending", False) + # resume_pending freshness ALSO uses the restart watchdog's ``last_resume_marked_at`` (the true + # interruption stamp): the transcript clock can be hours older for an active thread, and the + # startup auto-resume turn has empty text, so gating on it alone yields a blank user message. + mark_is_fresh = resume_pending and _is_fresh_gateway_interruption( + getattr(entry, "last_resume_marked_at", None), window_secs=window, ) - _has_fresh_tool_tail = bool( - agent_history - and agent_history[-1].get("role") == "tool" - and _interruption_is_fresh - ) - - if _is_resume_pending: - _reason = getattr(_resume_entry, "resume_reason", None) or "restart_timeout" - # Empty message = the startup auto-resume turn from _schedule_resume_pending_sessions; - # there is no NEW user message. Interactive platforms report the restore and ask what - # next; event platforms (webhook, API server) continue the work — nobody is present to - # answer, and an acknowledgement would silently abandon the task. - _resume_adapter = self._runner._adapter_for_source(ctx.source) - _interactive_resume = bool( - getattr(_resume_adapter, "interactive_resume", True) + if resume_pending and (interruption_is_fresh or mark_is_fresh): + reason = getattr(entry, "resume_reason", None) or "restart_timeout" + # Empty message = the startup auto-resume turn; there is no NEW user message. + ctx.message, persist_override = _prepare_resume_pending_message( + reason, ctx.message, interactive=self._resume_note_interactive(), ) - ctx.message, _persist_user_message_override = _prepare_resume_pending_message( - _reason, ctx.message, interactive=_interactive_resume, - ) - elif _has_fresh_tool_tail: - _persist_user_message_override = ctx.message + elif agent_history and agent_history[-1].get("role") == "tool" and interruption_is_fresh: + persist_override = ctx.message ctx.message = ( "[System note: A new message has arrived. The conversation " "history contains pending tool outputs from an interrupted turn. " @@ -1924,308 +1531,249 @@ class TurnRunner: "below FIRST. Do NOT re-execute old tool calls from the history.]\n\n" + ctx.message ) - - # Consume one-shot /reload-skills note (same queue pattern as CLI): prepend to the NEXT user - # message, then clear. Nothing hit the transcript out-of-band, so alternation stays intact. - _pending_notes = getattr(self._runner, "_pending_skills_reload_notes", None) - if _pending_notes and ctx.session_key and ctx.session_key in _pending_notes: - _srn = _pending_notes.pop(ctx.session_key, None) - if _srn: - ctx.message = _srn + "\n\n" + ctx.message - - # Safety net: a startup auto-resume event carries empty text and relies on the resume_pending - # branch above for the recovery note. If it did not fire (freshness signals disagreed, marker - # cleared before dispatch) we must NOT hand the model a blank user turn. Restricted to - # resume_pending sessions so legitimately empty turns (caption-less image) are untouched. - if ( - isinstance(ctx.message, str) - and not ctx.message.strip() - and _resume_entry is not None - and getattr(_resume_entry, "resume_pending", False) - ): - _sn_reason = ( - getattr(_resume_entry, "resume_reason", None) or "restart_timeout" - ) - _sn_adapter = self._runner._adapter_for_source(ctx.source) + self._prepend_pending_note("_pending_skills_reload_notes") + # Safety net: a startup auto-resume event carries empty text; if the resume_pending branch + # did not fire (freshness signals disagreed, marker cleared) we must NOT hand the model a blank + # user turn. Restricted to resume_pending sessions so caption-less image turns are untouched. + if isinstance(ctx.message, str) and not ctx.message.strip() and resume_pending: ctx.message = build_resume_recovery_note( - _sn_reason, - "", - interactive=bool( - getattr(_sn_adapter, "interactive_resume", True) - ), + getattr(entry, "resume_reason", None) or "restart_timeout", "", + interactive=self._resume_note_interactive(), ) - return _persist_user_message_override, _persist_user_timestamp_override + return persist_override, ctx.persist_user_timestamp - def _run_conversation_with_approval( - self, agent, agent_history, observed_group_context, - _persist_user_message_override, _persist_user_timestamp_override, - ): - from gateway.run import _wrap_current_message_with_observed_context + def _native_image_run_message(self): + """Wrap the user turn as an OpenAI-style multimodal content list when + _prepare_inbound_message_text buffered image paths; consume-and-clear so later turns on the + same runner never re-attach stale images. Falls back to plain text when nothing is readable.""" ctx = self._ctx - # Per-session gateway approval callback: dangerous-command approval blocks the agent thread - # (mirrors CLI input()); the callback bridges sync→async to send the request immediately. + native_imgs = self._runner._consume_pending_native_image_paths(ctx.session_key) + if not native_imgs: + return ctx.message + try: + from agent.image_routing import build_native_content_parts + parts, skipped = build_native_content_parts(ctx.message, native_imgs) + if skipped: + logger.warning("Native image attachment: skipped %d unreadable path(s): %s", len(skipped), skipped) + if any(p.get("type") == "image_url" for p in parts): + return parts + except Exception as exc: + logger.warning("Native image attachment failed, falling back to text: %s", exc) + return ctx.message + + def _run_conversation_with_approval(self, agent, agent_history, observed_group_context, + persist_user_message_override, persist_user_timestamp_override): + """Run the turn with the per-session gateway approval callback registered: dangerous-command + approval blocks the agent thread (mirrors CLI input()); the callback bridges sync→async.""" + from gateway.run import _wrap_current_message_with_observed_context from tools.approval import ( - register_gateway_notify, - reset_current_session_key, - set_current_session_key, + register_gateway_notify, reset_current_session_key, set_current_session_key, unregister_gateway_notify, ) - - _approval_session_key = ctx.session_key or "" - _approval_session_token = set_current_session_key(_approval_session_key) - register_gateway_notify(_approval_session_key, self._approval_notify_sync) + ctx = self._ctx + session_key = ctx.session_key or "" + token = set_current_session_key(session_key) + register_gateway_notify(session_key, self._approval_notify_sync) try: - # If _prepare_inbound_message_text buffered image paths for native attachment, wrap the - # user turn as an OpenAI-style multimodal content list. Consume-and-clear so subsequent - # turns on the same runner instance don't re-attach stale images. - _native_imgs = self._runner._consume_pending_native_image_paths(ctx.session_key) - if _native_imgs: - try: - from agent.image_routing import build_native_content_parts - _parts, _skipped = build_native_content_parts( - ctx.message, - _native_imgs, - ) - if _skipped: - logger.warning( - "Native image attachment: skipped %d unreadable path(s): %s", - len(_skipped), _skipped, - ) - if any(p.get("type") == "image_url" for p in _parts): - _run_message: Any = _parts - else: - # All images failed to read — fall back to plain text. - _run_message = ctx.message - except Exception as _img_exc: - logger.warning( - "Native image attachment failed, falling back to text: %s", - _img_exc, - ) - _run_message = ctx.message - else: - _run_message = ctx.message - - _api_run_message = _wrap_current_message_with_observed_context( - _run_message, - observed_group_context, + api_message = _wrap_current_message_with_observed_context( + self._native_image_run_message(), observed_group_context, ) - _conversation_kwargs = { - "conversation_history": agent_history, - "task_id": ctx.session_id, - } - if _persist_user_message_override is not None: - _conversation_kwargs["persist_user_message"] = _persist_user_message_override + kwargs = {"conversation_history": agent_history, "task_id": ctx.session_id} + if persist_user_message_override is not None: + kwargs["persist_user_message"] = persist_user_message_override elif observed_group_context: - _conversation_kwargs["persist_user_message"] = ctx.message + kwargs["persist_user_message"] = ctx.message if ctx.persist_user_display_kind: - # Internal self-injected turn: type the persisted user row at turn start so UIs - # render it as a timeline notice, not a user bubble. Role/content are untouched and - # the key is stripped from provider-bound payloads in conversation_loop. - _conversation_kwargs["persist_user_display_kind"] = ( - ctx.persist_user_display_kind - ) + # Internal self-injected turn: type the persisted user row so UIs render it as a + # timeline notice, not a user bubble (stripped from provider payloads downstream). + kwargs["persist_user_display_kind"] = ctx.persist_user_display_kind if ctx.moa_config is not None: - _conversation_kwargs["moa_config"] = ctx.moa_config - if _persist_user_timestamp_override is not None: - _conversation_kwargs["persist_user_timestamp"] = _persist_user_timestamp_override - # Thread the platform-side inbound message id onto the persisted user turn so a turn - # interrupted by a restart is recorded WITH its id — drain-window recovery dedups on - # has_platform_message_id. Uses the raw inbound id, NOT event_message_id (reply anchor). + kwargs["moa_config"] = ctx.moa_config + if persist_user_timestamp_override is not None: + kwargs["persist_user_timestamp"] = persist_user_timestamp_override + # The RAW inbound id (not event_message_id, the reply anchor) rides the persisted user + # turn so a restart-interrupted turn is recorded WITH its id for drain-window dedup. if ctx.inbound_message_id is not None: - _conversation_kwargs["persist_user_platform_id"] = str(ctx.inbound_message_id) - result = agent.run_conversation(_api_run_message, **_conversation_kwargs) + kwargs["persist_user_platform_id"] = str(ctx.inbound_message_id) + return agent.run_conversation(api_message, **kwargs) finally: - unregister_gateway_notify(_approval_session_key) - # Cancel any pending clarify entries so blocked agent threads don't hang past the end of - # the run (interrupt, completion, gateway shutdown). Idempotent. + unregister_gateway_notify(session_key) + # Cancel pending clarify entries so blocked agent threads don't hang past the end of the + # run (interrupt, completion, gateway shutdown). Idempotent. try: - from tools.clarify_gateway import clear_session as _clear_clarify_session - _clear_clarify_session(_approval_session_key) + from tools.clarify_gateway import clear_session + clear_session(session_key) except Exception: pass - reset_current_session_key(_approval_session_token) - return result + reset_current_session_key(token) - def _finish_stream_consumer(self, result, agent_history, _stream_consumer): + def _finish_stream_consumer(self, result, agent_history, stream_consumer): ctx = self._ctx - # Canonicalize a model-emitted computer-use screenshot path at the common result boundary: the - # streaming finalizer below and the non-streaming delivery path must see the same response; - # repairing only in later media scanning leaves streaming a mangled path + rejected attachment. - if isinstance(result, dict): - _result_final = result.get("final_response") - if isinstance(_result_final, str): - result["final_response"] = repair_explicit_computer_use_media_paths( - _result_final, - result.get("messages", []), - history_offset=len(agent_history), - ) - + # Canonicalize a model-emitted computer-use screenshot path at the common result boundary so + # the streaming finalizer and the non-streaming delivery path see the same response. + if isinstance(result, dict) and isinstance(result.get("final_response"), str): + result["final_response"] = repair_explicit_computer_use_media_paths( + result["final_response"], result.get("messages", []), history_offset=len(agent_history), + ) ctx.result_holder[0] = result + if stream_consumer is None: + return + # Pass final_response as the authoritative finalize payload: it includes post-stream + # augmentation (verifier footer, explainer) the accumulator never saw. Adopt ONLY a genuinely + # completed final: interrupt paths return {interrupted: True, completed: False} with a + # DIAGNOSTIC final_response — adopting it would seal the partial answer over with the + # diagnostic AND suppress the gateway's own error delivery. + _final_for_stream = None + if ( + isinstance(result, dict) + and not result.get("failed") + and not result.get("interrupted") + and result.get("completed") is not False + ): + fr = result.get("final_response") + if isinstance(fr, str) and fr.strip() and fr != "(empty)": + _final_for_stream = fr + if _final_for_stream is None: + stream_consumer.finish() + return + # Duck-type safe: test doubles / older consumers may expose a zero-arg finish(). + try: + stream_consumer.finish(_final_for_stream) + except TypeError: + stream_consumer.finish() - # Signal the stream consumer that the agent is done, passing final_response as the - # authoritative finalize payload: it includes post-stream augmentation (verifier footer, - # explainer) the accumulator never saw, so the seal delivers the TRUE final with no - # corrective send. Failed turns pass nothing — error text goes via the normal path. - if _stream_consumer is not None: - _final_for_stream = None - # Adopt ONLY a genuinely completed final: interrupt paths return {interrupted: True, - # completed: False} with a DIAGNOSTIC final_response and no failed key — adopting it - # would seal the streamed partial answer over with the diagnostic AND make - # delivered_final_matches reconcile, suppressing the gateway's own error delivery. - if ( - isinstance(result, dict) - and not result.get("failed") - and not result.get("interrupted") - and result.get("completed") is not False - ): - _fr = result.get("final_response") - if isinstance(_fr, str) and _fr.strip() and _fr != "(empty)": - _final_for_stream = _fr - if _final_for_stream is not None: - # Duck-type safe: test doubles / older consumers may expose a zero-arg finish(). The - # payload is an optimization, not a requirement — fall back to the bare signal. - try: - _stream_consumer.finish(_final_for_stream) - except TypeError: - _stream_consumer.finish() - else: - _stream_consumer.finish() + def _restore_telegram_thread_id_after_split(self, agent_session_id) -> None: + """Telegram DM whose source.thread_id was lost in the session split (synthetic/recovered + event): restore it from the binding so _thread_metadata_for_source yields the right + message_thread_id instead of the General thread (non-fatal).""" + ctx = self._ctx + try: + # run_sync is off-loop (executor); sync DB is fine. + binding = self._runner._session_db._db.get_telegram_topic_binding_by_session(session_id=agent_session_id) + if binding and binding.get("thread_id"): + ctx.source.thread_id = str(binding["thread_id"]) + logger.debug( + "Restored source.thread_id=%s from binding after session split %s → %s", + ctx.source.thread_id, ctx.session_id, agent_session_id, + ) + except Exception: + logger.debug("Failed to restore thread_id from binding after session split", exc_info=True) def _sync_session_after_run(self, agent_history): + """Sync session_id right after run_conversation(): compression can rotate before a + follow-up model call fails, and the failure return must still point at the compressed child. + + Returns (compacted_in_place, effective_session_id, effective_history_offset). + """ ctx = self._ctx - # Sync session_id right after run_conversation(): compression can rotate before a follow-up - # model call fails, and the failure return below must still point at the compressed child. + runner = self._runner agent = ctx.agent_holder[0] - _session_was_split = False - # In-place compaction (compression.in_place) compacts the transcript WITHOUT rotating the id, - # so the id-change diff below can't see it. compress_context() sets this flag on the agent; the - # gateway re-baselines (history_offset=0 + JSONL rewrite) as for a split despite unchanged id. - _compacted_in_place = bool(getattr(agent, "_last_compaction_in_place", False)) if agent else False + # In-place compaction compacts the transcript WITHOUT rotating the id, so the id-change diff + # can't see it; compress_context() sets this flag and the gateway re-baselines as for a split. + compacted_in_place = bool(getattr(agent, "_last_compaction_in_place", False)) if agent else False agent_session_id = getattr(agent, 'session_id', ctx.session_id) if agent else ctx.session_id - if agent and ctx.session_key and agent_session_id != ctx.session_id: - _session_was_split = True - logger.info( - "Session split detected: %s → %s (compression)", - ctx.session_id, agent_session_id, - ) - entry = self._runner.session_store._entries.get(ctx.session_key) - _session_split_entry_persisted = False + session_was_split = bool(agent and ctx.session_key and agent_session_id != ctx.session_id) + if session_was_split: + logger.info("Session split detected: %s → %s (compression)", ctx.session_id, agent_session_id) + entry = runner.session_store._entries.get(ctx.session_key) + persisted = False if entry: entry_session_id = getattr(entry, "session_id", None) if not ctx._run_still_current(): logger.info( "Skipping session split sync for stale run %s — " "generation %s is no longer current", - ctx.session_key or "?", - ctx.run_generation, + ctx.session_key or "?", ctx.run_generation, ) elif entry_session_id == agent_session_id: - _session_split_entry_persisted = True + persisted = True elif entry_session_id != ctx.session_id: logger.info( "Skipping session split sync for %s because the " "session binding moved from %s to %s before " "compression finished", - ctx.session_key or "?", - ctx.session_id, - entry_session_id, + ctx.session_key or "?", ctx.session_id, entry_session_id, ) else: entry.session_id = agent_session_id - self._runner.session_store._save() - self._runner.session_store._record_gateway_session_peer( - agent_session_id, - ctx.session_key, - ctx.source, - ) - _session_split_entry_persisted = True + runner.session_store._save() + runner.session_store._record_gateway_session_peer(agent_session_id, ctx.session_key, ctx.source) + persisted = True + # Only after this run published its split — a stale /stop→/new predecessor must not + # mutate routing state. + if persisted: + if ( + getattr(ctx.source, "platform", None) == Platform.TELEGRAM + and getattr(ctx.source, "chat_type", None) == "dm" + and getattr(ctx.source, "thread_id", None) is None + and runner._session_db is not None + ): + self._restore_telegram_thread_id_after_split(agent_session_id) + runner._sync_telegram_topic_binding(ctx.source, entry, reason="agent-run-compression") + runner._sync_session_model_from_agent(agent_session_id, agent) + # history_offset=0 whenever the agent's message list lost the original history prefix + # (split OR in-place compaction): the returned `messages` is the compacted set, persist all + # of it; slicing past the pre-compaction length would drop everything. + offset = 0 if (session_was_split or compacted_in_place) else len(agent_history) + return compacted_in_place, agent_session_id, offset - # Telegram DM whose source.thread_id was lost in the session split (synthetic/recovered - # event): restore it from the binding so _thread_metadata_for_source yields the right - # message_thread_id instead of the General thread (non-fatal). Only after this run - # published its split — a stale /stop→/new predecessor must not mutate routing state. - if _session_split_entry_persisted and ( - getattr(ctx.source, "platform", None) == Platform.TELEGRAM - and getattr(ctx.source, "chat_type", None) == "dm" - and getattr(ctx.source, "thread_id", None) is None - and self._runner._session_db is not None - ): - try: - # run_sync is off-loop (executor); sync DB is fine. - _binding = self._runner._session_db._db.get_telegram_topic_binding_by_session( - session_id=agent_session_id, - ) - if _binding and _binding.get("thread_id"): - ctx.source.thread_id = str(_binding["thread_id"]) - logger.debug( - "Restored source.thread_id=%s from binding after session split %s → %s", - ctx.source.thread_id, - ctx.session_id, - agent_session_id, - ) - except Exception: - logger.debug( - "Failed to restore thread_id from binding after session split", - exc_info=True, - ) - if _session_split_entry_persisted: - self._runner._sync_telegram_topic_binding( - ctx.source, entry, reason="agent-run-compression", - ) + def _combined_ephemeral_prompt(self) -> str: + """Platform context + YAML channel_prompts hint + channel_overrides system_prompt (or global + ephemeral) + the gateway ephemeral prompt.""" + ctx = self._ctx + combined = ctx.context_prompt or "" + for extra in ( + (ctx.channel_prompt or "").strip(), + self._runner._get_system_prompt_for_channel( + ctx.source.platform, ctx.source.chat_id or "", + thread_id=getattr(ctx.source, "thread_id", None), + parent_id=getattr(ctx.source, "parent_chat_id", None), + ), + ): + if extra: + combined = (combined + "\n\n" + extra).strip() + return combined - effective_session_id = agent_session_id - self._runner._sync_session_model_from_agent(effective_session_id, agent) - # history_offset=0 whenever the agent's message list lost the original history prefix: rotation - # (split) OR in-place compaction. Either way the returned `messages` is the compacted set, so - # persist all of it; slicing past the pre-compaction length would drop everything. - _effective_history_offset = ( - 0 if (_session_was_split or _compacted_in_place) else len(agent_history) + def _append_auto_media_tags(self, final_response: str, result, agent_history, history_media_paths) -> str: + """Append MEDIA: tags from tool results (e.g. TTS) that the model's final text omits, so + extract_media() delivers each file once. Scoped to THIS turn (slice at len(agent_history)) so + a stale MEDIA: path from an earlier turn never rides a later reply; the history-path dedup is + the secondary guard — and the sole one when mid-run compression shrank the list.""" + from gateway.run import _collect_auto_append_media_tags + if "MEDIA:" in final_response: + return final_response + media_tags, has_voice_directive = _collect_auto_append_media_tags( + result.get("messages", []), history_offset=len(agent_history), + history_media_paths=history_media_paths, ) - return _compacted_in_place, effective_session_id, _effective_history_offset + if not media_tags: + return final_response + unique_tags = list(dict.fromkeys(media_tags)) + if has_voice_directive: + unique_tags.insert(0, "[[audio_as_voice]]") + return final_response + "\n" + "\n".join(unique_tags) def run_sync(self): + """Executor-thread body of the turn; returns the gateway result dict. + + The turn message lives on the shared TurnContext (``ctx.message``), so the outer + ``_run_agent_inner`` sees every rebind. session_key propagates via contextvars + (_set_session_env / set_current_session_key) — never os.environ["HERMES_SESSION_KEY"], which + is process-global and would misroute approvals across concurrent sessions. + """ from gateway.run import ( - _collect_auto_append_media_tags, - _current_max_iterations, - _normalize_empty_agent_response, + _current_max_iterations, _normalize_empty_agent_response, _sanitize_gateway_final_response, ) ctx = self._ctx - # As a method the turn message lives on the shared TurnContext: every rebind writes - # `ctx.message`, so the outer `_run_agent_inner` body sees the update as via the closure cell. - - # session_key propagates via contextvars (_set_session_env / set_current_session_key): - # concurrency-safe and inherited by tool worker threads. Deliberately do NOT write - # os.environ["HERMES_SESSION_KEY"]: it is process-global, so concurrent sessions would clobber - # each other and a tool thread with an unset contextvar would read the wrong key, misrouting - # approvals. Only the TUI slash-worker subprocess exports the env var (from its own argv). - - # Map platform enum to the platform hint key the agent understands. - # Platform.LOCAL ("local") maps to "cli"; others pass through as-is. + runner = self._runner + # Platform.LOCAL ("local") maps to the "cli" hint key the agent understands. platform_key = "cli" if ctx.source.platform == Platform.LOCAL else ctx.source.platform.value - - # Combine platform context, YAML channel_prompts hint for this chat, channel_overrides - # system_prompt (or global ephemeral), and the gateway ephemeral prompt. - combined_ephemeral = ctx.context_prompt or "" - event_channel_prompt = (ctx.channel_prompt or "").strip() - if event_channel_prompt: - combined_ephemeral = (combined_ephemeral + "\n\n" + event_channel_prompt).strip() - cfg_channel_prompt = self._runner._get_system_prompt_for_channel( - ctx.source.platform, - ctx.source.chat_id or "", - thread_id=getattr(ctx.source, "thread_id", None), - parent_id=getattr(ctx.source, "parent_chat_id", None), - ) - if cfg_channel_prompt: - combined_ephemeral = (combined_ephemeral + "\n\n" + cfg_channel_prompt).strip() - + combined_ephemeral = self._combined_ephemeral_prompt() max_iterations = _current_max_iterations() - try: - model, runtime_kwargs = self._runner._resolve_session_agent_runtime( - source=ctx.source, - session_key=ctx.session_key, - user_config=ctx.user_config, + model, runtime_kwargs = runner._resolve_session_agent_runtime( + source=ctx.source, session_key=ctx.session_key, user_config=ctx.user_config, ) logger.debug( "run_agent resolved: model=%s provider=%s session=%s", @@ -2238,165 +1786,77 @@ class TurnRunner: "api_calls": 0, "tools": [], } - - pr = self._runner._provider_routing - reasoning_config = self._runner._resolve_session_reasoning_config( - source=ctx.source, - session_key=ctx.session_key, - model=model, + pr = runner._provider_routing + reasoning_config = runner._resolve_session_reasoning_config( + source=ctx.source, session_key=ctx.session_key, model=model, ) - self._runner._reasoning_config = reasoning_config - self._runner._service_tier = self._runner._resolve_session_service_tier( - source=ctx.source, session_key=ctx.session_key - ) - ( - _stream_consumer, - _stream_delta_cb, - _interim_assistant_cb, - _want_interim_messages, - ) = self._setup_stream_consumer(platform_key) - - turn_route = self._runner._resolve_turn_agent_config(ctx.message, model, runtime_kwargs) + runner._reasoning_config = reasoning_config + runner._service_tier = runner._resolve_session_service_tier(source=ctx.source, session_key=ctx.session_key) + stream_consumer, stream_delta_cb, interim_cb, want_interim = self._setup_stream_consumer(platform_key) + turn_route = runner._resolve_turn_agent_config(ctx.message, model, runtime_kwargs) agent, reused_cached_agent = self._resolve_turn_agent( turn_route, platform_key, combined_ephemeral, max_iterations, reasoning_config, pr, ) - self._wire_turn_agent_callbacks( - agent, turn_route, reasoning_config, - _stream_delta_cb, _interim_assistant_cb, _want_interim_messages, - ) - agent_history, observed_group_context, _history_media_paths = ( - self._load_turn_history(agent, reused_cached_agent) - ) - _persist_user_message_override, _persist_user_timestamp_override = ( - self._prepare_turn_message(agent_history) - ) - result = self._run_conversation_with_approval( - agent, agent_history, observed_group_context, - _persist_user_message_override, _persist_user_timestamp_override, - ) - self._finish_stream_consumer(result, agent_history, _stream_consumer) - - # Signal the streaming-TTS consumer that the agent is done. finish() runs on the outer - # event-loop thread after the executor returns, so early run_sync returns are also finalised. - - # Return final response, or a message if something went wrong + self._wire_turn_agent_callbacks(agent, turn_route, reasoning_config, stream_delta_cb, interim_cb, want_interim) + agent_history, observed_group_context, history_media_paths = self._load_turn_history(agent, reused_cached_agent) + persist_msg, persist_ts = self._prepare_turn_message(agent_history) + result = self._run_conversation_with_approval(agent, agent_history, observed_group_context, persist_msg, persist_ts) + self._finish_stream_consumer(result, agent_history, stream_consumer) + # The streaming-TTS consumer's finish() runs on the outer loop thread after the executor + # returns, so early run_sync returns are also finalised. final_response = result.get("final_response") - - # Extract actual token counts from the agent instance used for this run - _last_prompt_toks = 0 - _input_toks = 0 - _output_toks = 0 - _context_length = 0 - _agent = ctx.agent_holder[0] - if _agent and hasattr(_agent, "context_compressor"): - _last_prompt_toks = getattr(_agent.context_compressor, "last_prompt_tokens", 0) - _input_toks = getattr(_agent, "session_prompt_tokens", 0) - _output_toks = getattr(_agent, "session_completion_tokens", 0) - _context_length = getattr(_agent.context_compressor, "context_length", 0) or 0 - _resolved_model = getattr(_agent, "model", None) if _agent else None - - _compacted_in_place, effective_session_id, _effective_history_offset = ( - self._sync_session_after_run(agent_history) - ) - + # Actual token counts from the agent instance used for this run. + agent = ctx.agent_holder[0] + has_comp = bool(agent) and hasattr(agent, "context_compressor") + comp = agent.context_compressor if has_comp else None + usage = { + "last_prompt_tokens": getattr(comp, "last_prompt_tokens", 0) if has_comp else 0, + "input_tokens": getattr(agent, "session_prompt_tokens", 0) if has_comp else 0, + "output_tokens": getattr(agent, "session_completion_tokens", 0) if has_comp else 0, + "model": getattr(agent, "model", None) if agent else None, + "context_length": (getattr(comp, "context_length", 0) or 0) if has_comp else 0, + } + compacted_in_place, effective_session_id, history_offset = self._sync_session_after_run(agent_history) + common = { + "messages": result.get("messages", []), + "api_calls": result.get("api_calls", 0), + "failed": result.get("failed", False), + # The classifier's failure_reason must survive the empty-response path too, or + # downstream consumers (TUI billing, transient-failure persistence) lose it. + "failure_reason": result.get("failure_reason"), + "partial": result.get("partial", False), + "completed": result.get("completed"), + "interrupted": result.get("interrupted", False), + "interrupt_message": result.get("interrupt_message"), + "error": result.get("error"), + "compression_exhausted": result.get("compression_exhausted", False), + # Soft lock-contention defer: distinct from compression_exhausted so the gateway never + # auto-resets a session that a concurrent compressor is about to shrink. + "compression_deferred": result.get("compression_deferred", False), + "tools": ctx.tools_holder[0] or [], + "history_offset": history_offset, + "compacted_in_place": compacted_in_place, + "session_id": effective_session_id, + **usage, + } if not final_response: - final_response = _normalize_empty_agent_response( - result, final_response or "", history_len=len(agent_history), - ) + final_response = _normalize_empty_agent_response(result, final_response or "", history_len=len(agent_history)) final_response = _sanitize_gateway_final_response(ctx.source.platform, final_response) if not final_response: final_response = f"⚠️ {result['error']}" if result.get("error") else "" - return { - "final_response": final_response, - "messages": result.get("messages", []), - "api_calls": result.get("api_calls", 0), - "failed": result.get("failed", False), - # Sibling of the non-empty-response return below: the classifier's failure_reason - # must survive the empty-response path too, or downstream consumers (TUI billing, - # transient-failure persistence) lose the structured reason when no text was produced. - "failure_reason": result.get("failure_reason"), - "partial": result.get("partial", False), - "completed": result.get("completed"), - "interrupted": result.get("interrupted", False), - "interrupt_message": result.get("interrupt_message"), - "error": result.get("error"), - "compression_exhausted": result.get("compression_exhausted", False), - "compression_deferred": result.get("compression_deferred", False), - "tools": ctx.tools_holder[0] or [], - "history_offset": _effective_history_offset, - "compacted_in_place": _compacted_in_place, - "session_id": effective_session_id, - "last_prompt_tokens": _last_prompt_toks, - "input_tokens": _input_toks, - "output_tokens": _output_toks, - "model": _resolved_model, - "context_length": _context_length, - } - - # Append MEDIA: tags from tool results (e.g. TTS) that the model's final text omits, so - # extract_media() delivers each file once. Scope to THIS turn (slice at ``len(agent_history)``) - # so a stale MEDIA: path from an earlier turn doesn't ride a later text-only reply; dedup - # against _history_media_paths is the secondary guard — and the sole one on the fallback - # branch when mid-run compression shrank the list below the history length. - if "MEDIA:" not in final_response: - media_tags, has_voice_directive = _collect_auto_append_media_tags( - result.get("messages", []), - history_offset=len(agent_history), - history_media_paths=_history_media_paths, - ) - - if media_tags: - seen = set() - unique_tags = [] - for tag in media_tags: - if tag not in seen: - seen.add(tag) - unique_tags.append(tag) - if has_voice_directive: - unique_tags.insert(0, "[[audio_as_voice]]") - final_response = final_response + "\n" + "\n".join(unique_tags) - + # NOTE: deliberately omits agent_persisted/last_reasoning/response_* — the caller + # defaults agent_persisted differently when the key is absent. + return {"final_response": final_response, **common} + final_response = self._append_auto_media_tags(final_response, result, agent_history, history_media_paths) # Auto-titling runs at TURN START (agent/turn_context.py) from the user's message alone, so a - # failed/interrupted turn is still titled. Thread-rename callbacks are attached as - # `_on_session_title` before the run because the titler fires from the turn prologue. - + # failed/interrupted turn is still titled. return { "final_response": final_response, "last_reasoning": result.get("last_reasoning"), - "messages": ctx.result_holder[0].get("messages", []) if ctx.result_holder[0] else [], - "api_calls": ctx.result_holder[0].get("api_calls", 0) if ctx.result_holder[0] else 0, - "failed": ctx.result_holder[0].get("failed", False) if ctx.result_holder[0] else False, - "failure_reason": ( - ctx.result_holder[0].get("failure_reason") if ctx.result_holder[0] else None - ), - "completed": ctx.result_holder[0].get("completed") if ctx.result_holder[0] else None, - "interrupted": ctx.result_holder[0].get("interrupted", False) if ctx.result_holder[0] else False, - "partial": ctx.result_holder[0].get("partial", False) if ctx.result_holder[0] else False, - "error": ctx.result_holder[0].get("error") if ctx.result_holder[0] else None, - "interrupt_message": ctx.result_holder[0].get("interrupt_message") if ctx.result_holder[0] else None, - "compression_exhausted": ( - ctx.result_holder[0].get("compression_exhausted", False) - if ctx.result_holder[0] else False - ), - # Soft lock-contention defer: distinct from compression_exhausted so the gateway never - # auto-resets a session that a concurrent compressor is about to shrink. - "compression_deferred": ( - ctx.result_holder[0].get("compression_deferred", False) - if ctx.result_holder[0] else False - ), - "tools": ctx.tools_holder[0] or [], - "history_offset": _effective_history_offset, - "compacted_in_place": _compacted_in_place, - "last_prompt_tokens": _last_prompt_toks, - "input_tokens": _input_toks, - "output_tokens": _output_toks, - "model": _resolved_model, - "context_length": _context_length, - "session_id": effective_session_id, + **common, "response_previewed": result.get("response_previewed", False), "response_transformed": result.get("response_transformed", False), - # Pass through agent_persisted so the persistence block above can tell whether the codex - # app-server path self-persisted (it didn't — see codex_runtime.py); default True keeps the - # skip-db behaviour for the standard runtime. - "agent_persisted": (ctx.result_holder[0].get("agent_persisted", True) if ctx.result_holder[0] else True), + # Lets the persistence block tell whether the codex app-server path self-persisted (it + # didn't — see codex_runtime.py); default True keeps skip-db for the standard runtime. + "agent_persisted": result.get("agent_persisted", True), }