Teknium
66366d3dab
AST-driven, body-identical move of 359 GatewayRunner methods into cohesive
mixin modules (gateway/run_{voice,adapters,topics,turn,shutdown,busy,
config_loaders,startup,watchers,notifications,inbound,goals,agent_cache}.py)
plus TurnRunner -> gateway/run_turn_runner.py. run.py-internal symbols are
imported lazily inside method bodies so patch('gateway.run.X') keeps
intercepting; neutral deps are top-level; logger name stays 'gateway.run'.
_UNSET moved to leaf gateway/run_common.py (def-time default-arg sentinel).
Whole-module inspect.getsource(gateway_run) AST-walker tests repointed to
the module that now holds the walked code.
456 lines
21 KiB
Python
456 lines
21 KiB
Python
"""Session expiry / stall / catalog-refresh watcher loops for GatewayRunner.
|
|
|
|
Split out of ``gateway/run.py``; bound onto ``GatewayRunner`` via the MRO.
|
|
``gateway.run`` internals are imported lazily inside method bodies (import cycle),
|
|
so ``patch("gateway.run.X")`` keeps intercepting them at call time.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
from typing import TYPE_CHECKING
|
|
import asyncio
|
|
import time
|
|
from typing import Any, Dict, Optional
|
|
|
|
if TYPE_CHECKING: # string annotations only; never imported at runtime (cycle)
|
|
from gateway.run import GatewayRunner, TurnRunner # noqa: F401
|
|
|
|
# Log-record parity with the origin module.
|
|
logger = logging.getLogger("gateway.run")
|
|
|
|
|
|
class GatewaySessionWatchersMixin:
|
|
"""Session expiry / stall / catalog-refresh watcher loops for GatewayRunner."""
|
|
|
|
async def _session_expiry_watcher(self, interval: int = 300):
|
|
"""Background task that finalizes expired sessions: runs ``on_session_finalize`` hooks,
|
|
cleans up the cached agent's tool resources, evicts the cache entry, and marks the session
|
|
finalized so it is not finalized again.
|
|
"""
|
|
from gateway.run import _AGENT_PENDING_SENTINEL
|
|
await asyncio.sleep(60) # initial delay — let the gateway fully start
|
|
_finalize_failures: dict[str, int] = {} # session_id -> consecutive failure count
|
|
_MAX_FINALIZE_RETRIES = 3
|
|
while self._running:
|
|
try:
|
|
await self.async_session_store._ensure_loaded()
|
|
# Collect expired sessions first, then log a single summary.
|
|
_expired_entries = []
|
|
for key, entry in list(self.session_store._entries.items()):
|
|
if entry.expiry_finalized:
|
|
continue
|
|
if not await self.async_session_store._is_session_expired(entry):
|
|
continue
|
|
_expired_entries.append((key, entry))
|
|
|
|
if _expired_entries:
|
|
# Extract platform names from session keys for a compact summary.
|
|
# Keys look like "agent:main:telegram:dm:12345" — platform is field [2].
|
|
_platforms: dict[str, int] = {}
|
|
for _k, _e in _expired_entries:
|
|
_parts = _k.split(":")
|
|
_plat = _parts[2] if len(_parts) > 2 else "unknown"
|
|
_platforms[_plat] = _platforms.get(_plat, 0) + 1
|
|
_plat_summary = ", ".join(
|
|
f"{p}:{c}" for p, c in sorted(_platforms.items())
|
|
)
|
|
logger.info(
|
|
"Session expiry: %d sessions to finalize (%s)",
|
|
len(_expired_entries), _plat_summary,
|
|
)
|
|
|
|
for key, entry in _expired_entries:
|
|
try:
|
|
try:
|
|
_parts = key.split(":")
|
|
_platform = _parts[2] if len(_parts) > 2 else ""
|
|
# Off-loop + bounded: plugin finalize hooks can block arbitrarily, and
|
|
# this watcher runs on the gateway event loop.
|
|
await self._finalize_session_off_loop(
|
|
session_id=entry.session_id,
|
|
platform=_platform,
|
|
reason="session_expired",
|
|
)
|
|
except Exception:
|
|
pass
|
|
# Close the cached agent's memory provider and tool resources. Idle agents
|
|
# live in _agent_cache (not _running_agents), so look there.
|
|
_cached_agent = None
|
|
_cache_lock = getattr(self, "_agent_cache_lock", None)
|
|
if _cache_lock is not None:
|
|
with _cache_lock:
|
|
_cached = self._agent_cache.get(key)
|
|
_cached_agent = _cached[0] if isinstance(_cached, tuple) else _cached if _cached else None
|
|
# Fall back to _running_agents in case the agent is
|
|
# still mid-turn when the expiry fires.
|
|
if _cached_agent is None:
|
|
_exp_state = self._peek_session_state(key)
|
|
_cached_agent = _exp_state.turn.agent if _exp_state else None
|
|
if _cached_agent and _cached_agent is not _AGENT_PENDING_SENTINEL:
|
|
await self._cleanup_agent_resources_off_loop(
|
|
_cached_agent, context="session expiry"
|
|
)
|
|
# Drop the cache entry so the AIAgent (LLM clients, tool schemas, memory
|
|
# provider refs) can be GC'd; otherwise the cache grows unbounded.
|
|
self._evict_cached_agent(key)
|
|
# Permanent finalization: one funnel call drops every conversation-scoped
|
|
# dict AND boundary security state so they don't grow unbounded. Idle
|
|
# agent-cache eviction must NOT do this — that session is still alive and a
|
|
# resumed turn rebuilds from these overrides. Only finalize, /new, /reset clear.
|
|
self._clear_conversation_scope(
|
|
key, reason="expiry_finalized"
|
|
)
|
|
# Persist finalized flag (sessions.json AND state.db, single write-path);
|
|
# also drops the /model override — finalization is a conversation boundary.
|
|
await self.async_session_store.set_expiry_finalized(entry)
|
|
logger.debug(
|
|
"Session expiry finalized for %s",
|
|
entry.session_id,
|
|
)
|
|
_finalize_failures.pop(entry.session_id, None)
|
|
except Exception as e:
|
|
failures = _finalize_failures.get(entry.session_id, 0) + 1
|
|
_finalize_failures[entry.session_id] = failures
|
|
if failures >= _MAX_FINALIZE_RETRIES:
|
|
logger.warning(
|
|
"Session finalize gave up after %d attempts for %s: %s. "
|
|
"Marking as finalized to prevent infinite retry loop.",
|
|
failures, entry.session_id, e,
|
|
)
|
|
await self.async_session_store.set_expiry_finalized(
|
|
entry, clear_model_override=False
|
|
)
|
|
_finalize_failures.pop(entry.session_id, None)
|
|
else:
|
|
logger.debug(
|
|
"Session finalize failed (%d/%d) for %s: %s",
|
|
failures, _MAX_FINALIZE_RETRIES, entry.session_id, e,
|
|
)
|
|
|
|
if _expired_entries:
|
|
_done = sum(
|
|
1 for _, e in _expired_entries if e.expiry_finalized
|
|
)
|
|
_failed = len(_expired_entries) - _done
|
|
if _failed:
|
|
logger.info(
|
|
"Session expiry done: %d finalized, %d pending retry",
|
|
_done, _failed,
|
|
)
|
|
else:
|
|
logger.info(
|
|
"Session expiry done: %d finalized", _done,
|
|
)
|
|
|
|
# Sweep agents idle beyond the TTL regardless of session reset policy: sessions with
|
|
# long / "never" reset windows would otherwise pin memory for the gateway's life.
|
|
try:
|
|
_idle_evicted = self._sweep_idle_cached_agents()
|
|
if _idle_evicted:
|
|
logger.info(
|
|
"Agent cache idle sweep: evicted %d agent(s)",
|
|
_idle_evicted,
|
|
)
|
|
except Exception as _e:
|
|
logger.debug("Idle agent sweep failed: %s", _e)
|
|
|
|
# Neither LRU cap nor idle TTL knows what a cached transcript costs in memory, so a
|
|
# busy gateway keeps every warm session's tool output resident until the RSS limit.
|
|
try:
|
|
self._sweep_agent_cache_under_pressure()
|
|
except Exception as _e:
|
|
logger.debug("Agent cache pressure sweep failed: %s", _e)
|
|
|
|
# Prune stale SessionStore entries; the in-memory dict (and sessions.json) would
|
|
# otherwise grow unbounded with many rotating chats / threads / users.
|
|
_last_prune_ts = getattr(self, "_last_session_store_prune_ts", 0.0)
|
|
_prune_interval = 3600.0 # once per hour
|
|
if time.time() - _last_prune_ts > _prune_interval:
|
|
try:
|
|
_max_age = int(
|
|
getattr(self.config, "session_store_max_age_days", 0) or 0
|
|
)
|
|
if _max_age > 0:
|
|
_pruned = await self.async_session_store.prune_old_entries(_max_age)
|
|
if _pruned:
|
|
logger.info(
|
|
"SessionStore prune: dropped %d stale entries",
|
|
_pruned,
|
|
)
|
|
except Exception as _e:
|
|
logger.debug("SessionStore prune failed: %s", _e)
|
|
self._last_session_store_prune_ts = time.time()
|
|
except Exception as e:
|
|
logger.debug("Session expiry watcher error: %s", e)
|
|
# Sleep in small increments so we can stop quickly
|
|
for _ in range(interval):
|
|
if not self._running:
|
|
break
|
|
await asyncio.sleep(1)
|
|
|
|
def _session_stall_timeout_seconds(self) -> float:
|
|
"""Return configured stall timeout (seconds); 0 disables the watchdog."""
|
|
from gateway.run import _float_env
|
|
return _float_env("HERMES_SESSION_STALL_TIMEOUT", 300)
|
|
|
|
def _iter_gateway_adapters(self):
|
|
"""Yield every live platform adapter (default + multiplex profiles)."""
|
|
seen: set[int] = set()
|
|
for adapter in list(getattr(self, "adapters", {}).values()):
|
|
if adapter is None:
|
|
continue
|
|
aid = id(adapter)
|
|
if aid in seen:
|
|
continue
|
|
seen.add(aid)
|
|
yield adapter
|
|
for amap in list(getattr(self, "_profile_adapters", {}).values()):
|
|
for adapter in list(amap.values()):
|
|
if adapter is None:
|
|
continue
|
|
aid = id(adapter)
|
|
if aid in seen:
|
|
continue
|
|
seen.add(aid)
|
|
yield adapter
|
|
|
|
def _session_activity_for_stall(self, session_key: str) -> Optional[dict]:
|
|
"""Return the shared activity snapshot for stall progress: the single source is
|
|
``AIAgent.get_activity_summary()`` / ``agent.session_activity``; no turn-start or
|
|
pending-inbound clocks.
|
|
"""
|
|
from gateway.run import _AGENT_PENDING_SENTINEL
|
|
agent = (getattr(self, "_running_agents", None) or {}).get(session_key)
|
|
if agent is None or agent is _AGENT_PENDING_SENTINEL:
|
|
return None
|
|
if not hasattr(agent, "get_activity_summary"):
|
|
return None
|
|
try:
|
|
summary = agent.get_activity_summary()
|
|
except Exception:
|
|
return None
|
|
return summary if isinstance(summary, dict) else None
|
|
|
|
async def _check_session_stalls(self, timeout_seconds: float) -> int:
|
|
"""Scan pending inbound sessions and notify once per stall episode; returns the number of
|
|
notifications sent this pass (for tests).
|
|
"""
|
|
from gateway.run import _STALL_NOTIFY_SEND_TIMEOUT_SECONDS
|
|
from gateway.session_stall import (
|
|
format_session_stall_notification,
|
|
resolve_session_idle_seconds_from_activity,
|
|
should_clear_session_stall_notification,
|
|
should_emit_session_stall_notification,
|
|
)
|
|
|
|
notified_map = getattr(self, "_session_stall_notified", None)
|
|
if notified_map is None:
|
|
notified_map = {}
|
|
self._session_stall_notified = notified_map
|
|
|
|
sent = 0
|
|
now = time.time()
|
|
candidates: Dict[str, tuple[Any, Any]] = {}
|
|
|
|
for adapter in self._iter_gateway_adapters():
|
|
pending_slot = getattr(adapter, "_pending_messages", None) or {}
|
|
for session_key, event in list(pending_slot.items()):
|
|
if session_key and session_key not in candidates and event is not None:
|
|
candidates[session_key] = (adapter, event)
|
|
|
|
for session_key, overflow in list(
|
|
(getattr(self, "_queued_events", None) or {}).items()
|
|
):
|
|
if not session_key or session_key in candidates or not overflow:
|
|
continue
|
|
event = overflow[0]
|
|
source = getattr(event, "source", None)
|
|
adapter = (
|
|
self._adapter_for_source(source) if source is not None else None
|
|
)
|
|
if adapter is None:
|
|
continue
|
|
candidates[session_key] = (adapter, event)
|
|
|
|
for session_key, (adapter, pending_event) in list(candidates.items()):
|
|
has_pending = pending_event is not None
|
|
activity = (
|
|
self._session_activity_for_stall(session_key) if has_pending else None
|
|
)
|
|
idle_seconds = (
|
|
resolve_session_idle_seconds_from_activity(activity, now=now)
|
|
if has_pending
|
|
else None
|
|
)
|
|
already = bool(notified_map.get(session_key))
|
|
if should_clear_session_stall_notification(
|
|
timeout_seconds=timeout_seconds,
|
|
idle_seconds=idle_seconds,
|
|
has_pending_inbound=has_pending,
|
|
):
|
|
notified_map.pop(session_key, None)
|
|
already = False
|
|
if not should_emit_session_stall_notification(
|
|
timeout_seconds=timeout_seconds,
|
|
idle_seconds=idle_seconds,
|
|
has_pending_inbound=has_pending,
|
|
already_notified=already,
|
|
):
|
|
continue
|
|
|
|
if idle_seconds is None:
|
|
continue
|
|
mins = max(1, int(idle_seconds // 60))
|
|
activity = activity or {}
|
|
logger.warning(
|
|
"Session stall detected: session=%s idle=%.0fs "
|
|
"(timeout=%.0fs, ~%d min); pending inbound present "
|
|
"| last_activity=%s | provenance=%s "
|
|
"(agent.session_stall_timeout)",
|
|
session_key,
|
|
idle_seconds,
|
|
timeout_seconds,
|
|
mins,
|
|
activity.get("last_activity_desc")
|
|
or activity.get("last_activity_description")
|
|
or "unknown",
|
|
activity.get("provenance")
|
|
or activity.get("last_activity_provenance")
|
|
or "unknown",
|
|
)
|
|
source = getattr(pending_event, "source", None)
|
|
chat_id = getattr(source, "chat_id", None) if source is not None else None
|
|
if not chat_id:
|
|
logger.warning(
|
|
"Session stall notify skipped (no chat_id): session=%s",
|
|
session_key,
|
|
)
|
|
# Cannot deliver; latch to avoid log spam every tick.
|
|
notified_map[session_key] = True
|
|
continue
|
|
# Re-read pending state + activity IMMEDIATELY before delivery: the snapshot above ages
|
|
# while earlier candidates await sends; an agent that progressed (or drained its queue)
|
|
# must not get a false stall notice. Abort, latch un-set, so the next tick re-evaluates.
|
|
still_pending = (
|
|
(getattr(adapter, "_pending_messages", None) or {}).get(
|
|
session_key
|
|
)
|
|
is not None
|
|
or bool(
|
|
(getattr(self, "_queued_events", None) or {}).get(
|
|
session_key
|
|
)
|
|
)
|
|
)
|
|
fresh_idle = resolve_session_idle_seconds_from_activity(
|
|
self._session_activity_for_stall(session_key),
|
|
now=time.time(),
|
|
)
|
|
if not still_pending or (
|
|
fresh_idle is not None and fresh_idle < timeout_seconds
|
|
):
|
|
logger.info(
|
|
"Session stall notify aborted (no longer stale): "
|
|
"session=%s pending=%s fresh_idle=%s",
|
|
session_key,
|
|
still_pending,
|
|
fresh_idle,
|
|
)
|
|
# Re-arm: drop any stale latch so a FUTURE genuine stall
|
|
# episode notifies again.
|
|
notified_map.pop(session_key, None)
|
|
continue
|
|
try:
|
|
metadata = (
|
|
self._thread_metadata_for_source(source)
|
|
if source is not None and hasattr(self, "_thread_metadata_for_source")
|
|
else None
|
|
)
|
|
# Bound the send: a wedged adapter transport (network hang, dead websocket) must not
|
|
# block the watcher pass — siblings would go unevaluated and the watcher stop.
|
|
try:
|
|
result = await asyncio.wait_for(
|
|
adapter.send(
|
|
str(chat_id),
|
|
format_session_stall_notification(idle_seconds),
|
|
metadata=metadata,
|
|
),
|
|
timeout=_STALL_NOTIFY_SEND_TIMEOUT_SECONDS,
|
|
)
|
|
except asyncio.TimeoutError:
|
|
logger.warning(
|
|
"Session stall notify send timed out after %.0fs "
|
|
"for %s; will retry next tick",
|
|
_STALL_NOTIFY_SEND_TIMEOUT_SECONDS,
|
|
session_key,
|
|
)
|
|
continue # do not latch; retry next tick
|
|
# Adapters often return SendResult(success=False) instead of raising.
|
|
if result is not None and getattr(result, "success", True) is False:
|
|
logger.warning(
|
|
"Session stall notify failed for %s: %s",
|
|
session_key,
|
|
getattr(result, "error", "send returned success=False"),
|
|
)
|
|
continue # do not latch; retry next tick
|
|
sent += 1
|
|
notified_map[session_key] = True
|
|
except Exception as exc:
|
|
logger.warning(
|
|
"Session stall notify failed for %s: %s",
|
|
session_key,
|
|
exc,
|
|
)
|
|
# Do not latch — retry next watcher tick until delivery or episode clear.
|
|
|
|
# Drop latches for sessions that no longer appear in any pending map.
|
|
for key in list(notified_map.keys()):
|
|
if key not in candidates:
|
|
notified_map.pop(key, None)
|
|
|
|
return sent
|
|
|
|
async def _model_catalog_refresh_watcher(self) -> None:
|
|
"""Refresh the /model picker's remote catalogs every TTL window. The picker itself only
|
|
refreshes on a cold/stale open, so if nobody opens ``/model`` the cache never updates.
|
|
"""
|
|
from hermes_cli.model_catalog import refresh_catalogs, refresh_interval_seconds
|
|
|
|
await asyncio.sleep(30) # let startup settle
|
|
while self._running:
|
|
try:
|
|
await asyncio.to_thread(refresh_catalogs)
|
|
except Exception as exc:
|
|
logger.debug("Model catalog refresh failed: %s", exc)
|
|
try:
|
|
interval = refresh_interval_seconds()
|
|
except Exception:
|
|
interval = 1200.0
|
|
deadline = time.monotonic() + interval
|
|
while self._running and time.monotonic() < deadline:
|
|
await asyncio.sleep(min(30.0, max(0.0, deadline - time.monotonic())))
|
|
|
|
async def _session_stall_watcher(self, interval: float = 30.0):
|
|
"""Periodic pending-inbound + stale-activity stall watchdog.
|
|
|
|
Progress comes only from ``get_activity_summary()``. Pending inbound is a notify policy
|
|
gate, not a progress clock. Notify-only: does not kill the turn (contrast
|
|
``gateway_timeout`` / ``shutdown_watchdog``).
|
|
"""
|
|
# Short initial delay so startup reconnect noise does not false-fire.
|
|
await asyncio.sleep(min(30.0, max(1.0, float(interval))))
|
|
while self._running:
|
|
try:
|
|
timeout = self._session_stall_timeout_seconds()
|
|
if timeout > 0:
|
|
await self._check_session_stalls(timeout)
|
|
except Exception as exc:
|
|
logger.debug("Session stall watcher error: %s", exc)
|
|
# Interruptible sleep
|
|
steps = max(1, int(float(interval)))
|
|
for _ in range(steps):
|
|
if not self._running:
|
|
break
|
|
await asyncio.sleep(1)
|