"""Session expiry / stall / catalog-refresh watcher loops for GatewayRunner. Split out of ``gateway/run.py``; bound onto ``GatewayRunner`` via the MRO. ``gateway.run`` internals are imported lazily inside method bodies (import cycle), so ``patch("gateway.run.X")`` keeps intercepting them at call time. """ from __future__ import annotations import logging from typing import TYPE_CHECKING import asyncio import time from typing import Any, Dict, Optional if TYPE_CHECKING: # string annotations only; never imported at runtime (cycle) from gateway.run import GatewayRunner, TurnRunner # noqa: F401 # Log-record parity with the origin module. logger = logging.getLogger("gateway.run") class GatewaySessionWatchersMixin: """Session expiry / stall / catalog-refresh watcher loops for GatewayRunner.""" async def _session_expiry_watcher(self, interval: int = 300): """Background task that finalizes expired sessions: runs ``on_session_finalize`` hooks, cleans up the cached agent's tool resources, evicts the cache entry, and marks the session finalized so it is not finalized again. """ from gateway.run import _AGENT_PENDING_SENTINEL await asyncio.sleep(60) # initial delay — let the gateway fully start _finalize_failures: dict[str, int] = {} # session_id -> consecutive failure count _MAX_FINALIZE_RETRIES = 3 while self._running: try: await self.async_session_store._ensure_loaded() # Collect expired sessions first, then log a single summary. _expired_entries = [] for key, entry in list(self.session_store._entries.items()): if entry.expiry_finalized: continue if not await self.async_session_store._is_session_expired(entry): continue _expired_entries.append((key, entry)) if _expired_entries: # Extract platform names from session keys for a compact summary. # Keys look like "agent:main:telegram:dm:12345" — platform is field [2]. _platforms: dict[str, int] = {} for _k, _e in _expired_entries: _parts = _k.split(":") _plat = _parts[2] if len(_parts) > 2 else "unknown" _platforms[_plat] = _platforms.get(_plat, 0) + 1 _plat_summary = ", ".join( f"{p}:{c}" for p, c in sorted(_platforms.items()) ) logger.info( "Session expiry: %d sessions to finalize (%s)", len(_expired_entries), _plat_summary, ) for key, entry in _expired_entries: try: try: _parts = key.split(":") _platform = _parts[2] if len(_parts) > 2 else "" # Off-loop + bounded: plugin finalize hooks can block arbitrarily, and # this watcher runs on the gateway event loop. await self._finalize_session_off_loop( session_id=entry.session_id, platform=_platform, reason="session_expired", ) except Exception: pass # Close the cached agent's memory provider and tool resources. Idle agents # live in _agent_cache (not _running_agents), so look there. _cached_agent = None _cache_lock = getattr(self, "_agent_cache_lock", None) if _cache_lock is not None: with _cache_lock: _cached = self._agent_cache.get(key) _cached_agent = _cached[0] if isinstance(_cached, tuple) else _cached if _cached else None # Fall back to _running_agents in case the agent is # still mid-turn when the expiry fires. if _cached_agent is None: _exp_state = self._peek_session_state(key) _cached_agent = _exp_state.turn.agent if _exp_state else None if _cached_agent and _cached_agent is not _AGENT_PENDING_SENTINEL: await self._cleanup_agent_resources_off_loop( _cached_agent, context="session expiry" ) # Drop the cache entry so the AIAgent (LLM clients, tool schemas, memory # provider refs) can be GC'd; otherwise the cache grows unbounded. self._evict_cached_agent(key) # Permanent finalization: one funnel call drops every conversation-scoped # dict AND boundary security state so they don't grow unbounded. Idle # agent-cache eviction must NOT do this — that session is still alive and a # resumed turn rebuilds from these overrides. Only finalize, /new, /reset clear. self._clear_conversation_scope( key, reason="expiry_finalized" ) # Persist finalized flag (sessions.json AND state.db, single write-path); # also drops the /model override — finalization is a conversation boundary. await self.async_session_store.set_expiry_finalized(entry) logger.debug( "Session expiry finalized for %s", entry.session_id, ) _finalize_failures.pop(entry.session_id, None) except Exception as e: failures = _finalize_failures.get(entry.session_id, 0) + 1 _finalize_failures[entry.session_id] = failures if failures >= _MAX_FINALIZE_RETRIES: logger.warning( "Session finalize gave up after %d attempts for %s: %s. " "Marking as finalized to prevent infinite retry loop.", failures, entry.session_id, e, ) await self.async_session_store.set_expiry_finalized( entry, clear_model_override=False ) _finalize_failures.pop(entry.session_id, None) else: logger.debug( "Session finalize failed (%d/%d) for %s: %s", failures, _MAX_FINALIZE_RETRIES, entry.session_id, e, ) if _expired_entries: _done = sum( 1 for _, e in _expired_entries if e.expiry_finalized ) _failed = len(_expired_entries) - _done if _failed: logger.info( "Session expiry done: %d finalized, %d pending retry", _done, _failed, ) else: logger.info( "Session expiry done: %d finalized", _done, ) # Sweep agents idle beyond the TTL regardless of session reset policy: sessions with # long / "never" reset windows would otherwise pin memory for the gateway's life. try: _idle_evicted = self._sweep_idle_cached_agents() if _idle_evicted: logger.info( "Agent cache idle sweep: evicted %d agent(s)", _idle_evicted, ) except Exception as _e: logger.debug("Idle agent sweep failed: %s", _e) # Neither LRU cap nor idle TTL knows what a cached transcript costs in memory, so a # busy gateway keeps every warm session's tool output resident until the RSS limit. try: self._sweep_agent_cache_under_pressure() except Exception as _e: logger.debug("Agent cache pressure sweep failed: %s", _e) # Prune stale SessionStore entries; the in-memory dict (and sessions.json) would # otherwise grow unbounded with many rotating chats / threads / users. _last_prune_ts = getattr(self, "_last_session_store_prune_ts", 0.0) _prune_interval = 3600.0 # once per hour if time.time() - _last_prune_ts > _prune_interval: try: _max_age = int( getattr(self.config, "session_store_max_age_days", 0) or 0 ) if _max_age > 0: _pruned = await self.async_session_store.prune_old_entries(_max_age) if _pruned: logger.info( "SessionStore prune: dropped %d stale entries", _pruned, ) except Exception as _e: logger.debug("SessionStore prune failed: %s", _e) self._last_session_store_prune_ts = time.time() except Exception as e: logger.debug("Session expiry watcher error: %s", e) # Sleep in small increments so we can stop quickly for _ in range(interval): if not self._running: break await asyncio.sleep(1) def _session_stall_timeout_seconds(self) -> float: """Return configured stall timeout (seconds); 0 disables the watchdog.""" from gateway.run import _float_env return _float_env("HERMES_SESSION_STALL_TIMEOUT", 300) def _iter_gateway_adapters(self): """Yield every live platform adapter (default + multiplex profiles).""" seen: set[int] = set() for adapter in list(getattr(self, "adapters", {}).values()): if adapter is None: continue aid = id(adapter) if aid in seen: continue seen.add(aid) yield adapter for amap in list(getattr(self, "_profile_adapters", {}).values()): for adapter in list(amap.values()): if adapter is None: continue aid = id(adapter) if aid in seen: continue seen.add(aid) yield adapter def _session_activity_for_stall(self, session_key: str) -> Optional[dict]: """Return the shared activity snapshot for stall progress: the single source is ``AIAgent.get_activity_summary()`` / ``agent.session_activity``; no turn-start or pending-inbound clocks. """ from gateway.run import _AGENT_PENDING_SENTINEL agent = (getattr(self, "_running_agents", None) or {}).get(session_key) if agent is None or agent is _AGENT_PENDING_SENTINEL: return None if not hasattr(agent, "get_activity_summary"): return None try: summary = agent.get_activity_summary() except Exception: return None return summary if isinstance(summary, dict) else None async def _check_session_stalls(self, timeout_seconds: float) -> int: """Scan pending inbound sessions and notify once per stall episode; returns the number of notifications sent this pass (for tests). """ from gateway.run import _STALL_NOTIFY_SEND_TIMEOUT_SECONDS from gateway.session_stall import ( format_session_stall_notification, resolve_session_idle_seconds_from_activity, should_clear_session_stall_notification, should_emit_session_stall_notification, ) notified_map = getattr(self, "_session_stall_notified", None) if notified_map is None: notified_map = {} self._session_stall_notified = notified_map sent = 0 now = time.time() candidates: Dict[str, tuple[Any, Any]] = {} for adapter in self._iter_gateway_adapters(): pending_slot = getattr(adapter, "_pending_messages", None) or {} for session_key, event in list(pending_slot.items()): if session_key and session_key not in candidates and event is not None: candidates[session_key] = (adapter, event) for session_key, overflow in list( (getattr(self, "_queued_events", None) or {}).items() ): if not session_key or session_key in candidates or not overflow: continue event = overflow[0] source = getattr(event, "source", None) adapter = ( self._adapter_for_source(source) if source is not None else None ) if adapter is None: continue candidates[session_key] = (adapter, event) for session_key, (adapter, pending_event) in list(candidates.items()): has_pending = pending_event is not None activity = ( self._session_activity_for_stall(session_key) if has_pending else None ) idle_seconds = ( resolve_session_idle_seconds_from_activity(activity, now=now) if has_pending else None ) already = bool(notified_map.get(session_key)) if should_clear_session_stall_notification( timeout_seconds=timeout_seconds, idle_seconds=idle_seconds, has_pending_inbound=has_pending, ): notified_map.pop(session_key, None) already = False if not should_emit_session_stall_notification( timeout_seconds=timeout_seconds, idle_seconds=idle_seconds, has_pending_inbound=has_pending, already_notified=already, ): continue if idle_seconds is None: continue mins = max(1, int(idle_seconds // 60)) activity = activity or {} logger.warning( "Session stall detected: session=%s idle=%.0fs " "(timeout=%.0fs, ~%d min); pending inbound present " "| last_activity=%s | provenance=%s " "(agent.session_stall_timeout)", session_key, idle_seconds, timeout_seconds, mins, activity.get("last_activity_desc") or activity.get("last_activity_description") or "unknown", activity.get("provenance") or activity.get("last_activity_provenance") or "unknown", ) source = getattr(pending_event, "source", None) chat_id = getattr(source, "chat_id", None) if source is not None else None if not chat_id: logger.warning( "Session stall notify skipped (no chat_id): session=%s", session_key, ) # Cannot deliver; latch to avoid log spam every tick. notified_map[session_key] = True continue # Re-read pending state + activity IMMEDIATELY before delivery: the snapshot above ages # while earlier candidates await sends; an agent that progressed (or drained its queue) # must not get a false stall notice. Abort, latch un-set, so the next tick re-evaluates. still_pending = ( (getattr(adapter, "_pending_messages", None) or {}).get( session_key ) is not None or bool( (getattr(self, "_queued_events", None) or {}).get( session_key ) ) ) fresh_idle = resolve_session_idle_seconds_from_activity( self._session_activity_for_stall(session_key), now=time.time(), ) if not still_pending or ( fresh_idle is not None and fresh_idle < timeout_seconds ): logger.info( "Session stall notify aborted (no longer stale): " "session=%s pending=%s fresh_idle=%s", session_key, still_pending, fresh_idle, ) # Re-arm: drop any stale latch so a FUTURE genuine stall # episode notifies again. notified_map.pop(session_key, None) continue try: metadata = ( self._thread_metadata_for_source(source) if source is not None and hasattr(self, "_thread_metadata_for_source") else None ) # Bound the send: a wedged adapter transport (network hang, dead websocket) must not # block the watcher pass — siblings would go unevaluated and the watcher stop. try: result = await asyncio.wait_for( adapter.send( str(chat_id), format_session_stall_notification(idle_seconds), metadata=metadata, ), timeout=_STALL_NOTIFY_SEND_TIMEOUT_SECONDS, ) except asyncio.TimeoutError: logger.warning( "Session stall notify send timed out after %.0fs " "for %s; will retry next tick", _STALL_NOTIFY_SEND_TIMEOUT_SECONDS, session_key, ) continue # do not latch; retry next tick # Adapters often return SendResult(success=False) instead of raising. if result is not None and getattr(result, "success", True) is False: logger.warning( "Session stall notify failed for %s: %s", session_key, getattr(result, "error", "send returned success=False"), ) continue # do not latch; retry next tick sent += 1 notified_map[session_key] = True except Exception as exc: logger.warning( "Session stall notify failed for %s: %s", session_key, exc, ) # Do not latch — retry next watcher tick until delivery or episode clear. # Drop latches for sessions that no longer appear in any pending map. for key in list(notified_map.keys()): if key not in candidates: notified_map.pop(key, None) return sent async def _model_catalog_refresh_watcher(self) -> None: """Refresh the /model picker's remote catalogs every TTL window. The picker itself only refreshes on a cold/stale open, so if nobody opens ``/model`` the cache never updates. """ from hermes_cli.model_catalog import refresh_catalogs, refresh_interval_seconds await asyncio.sleep(30) # let startup settle while self._running: try: await asyncio.to_thread(refresh_catalogs) except Exception as exc: logger.debug("Model catalog refresh failed: %s", exc) try: interval = refresh_interval_seconds() except Exception: interval = 1200.0 deadline = time.monotonic() + interval while self._running and time.monotonic() < deadline: await asyncio.sleep(min(30.0, max(0.0, deadline - time.monotonic()))) async def _session_stall_watcher(self, interval: float = 30.0): """Periodic pending-inbound + stale-activity stall watchdog. Progress comes only from ``get_activity_summary()``. Pending inbound is a notify policy gate, not a progress clock. Notify-only: does not kill the turn (contrast ``gateway_timeout`` / ``shutdown_watchdog``). """ # Short initial delay so startup reconnect noise does not false-fire. await asyncio.sleep(min(30.0, max(1.0, float(interval)))) while self._running: try: timeout = self._session_stall_timeout_seconds() if timeout > 0: await self._check_session_stalls(timeout) except Exception as exc: logger.debug("Session stall watcher error: %s", exc) # Interruptible sleep steps = max(1, int(float(interval))) for _ in range(steps): if not self._running: break await asyncio.sleep(1)