"""Agent cache, session model overrides, turn leases, run generations and conversation-scope reset methods for GatewayRunner. Split out of ``gateway/run.py``; bound onto ``GatewayRunner`` via the MRO. ``gateway.run`` internals are imported lazily inside method bodies (import cycle), so ``patch("gateway.run.X")`` keeps intercepting them at call time. """ from __future__ import annotations import logging from typing import TYPE_CHECKING import inspect import threading import time from contextlib import suppress from gateway.config import Platform from gateway.session import SessionSource, build_session_context_prompt from hermes_cli.config import cfg_get from typing import Any, Dict, List, Optional if TYPE_CHECKING: # string annotations only; never imported at runtime (cycle) from gateway.run import GatewayRunner, TurnRunner # noqa: F401 # Log-record parity with the origin module. logger = logging.getLogger("gateway.run") class GatewayAgentCacheMixin: """Agent cache, session model overrides, turn leases, run generations and conversation-scope reset methods for GatewayRunner.""" @classmethod def _empty_honcho_cache_busting_config(cls) -> dict[str, Any]: return {key: None for key in cls._HONCHO_CACHE_BUSTING_KEYS} @classmethod def _extract_honcho_cache_busting_config(cls) -> dict[str, Any]: """Extract Honcho identity keys, memoized by honcho.json mtime.""" try: from plugins.memory.honcho.client import HonchoClientConfig, resolve_config_path path = resolve_config_path() try: mtime_ns = path.stat().st_mtime_ns except OSError: mtime_ns = None memo_key = (str(path), mtime_ns) cached = cls._HONCHO_CACHE_BUSTING_MEMO.get(memo_key) if cached is not None: return dict(cached) hcfg = HonchoClientConfig.from_global_config(config_path=path) aliases = hcfg.user_peer_aliases or {} values = { "honcho.peer_name": hcfg.peer_name, "honcho.ai_peer": hcfg.ai_peer, "honcho.pin_peer_name": bool(hcfg.pin_peer_name), "honcho.runtime_peer_prefix": hcfg.runtime_peer_prefix or "", "honcho.user_peer_aliases": sorted(aliases.items()) if isinstance(aliases, dict) else [], } cls._HONCHO_CACHE_BUSTING_MEMO = {memo_key: values} return dict(values) except Exception: return cls._empty_honcho_cache_busting_config() @classmethod def _extract_cache_busting_config(cls, user_config: dict | None) -> dict: """Pull values that must bust the cached agent, as a flat dict keyed by 'section.key'. Missing keys / non-dict sections yield None, which still enters the signature ('absent' vs 'present-and-null' differ). Includes the live tool registry generation: MCP reloads mutate the registry without touching config.yaml, and cached agents freeze their tool schemas. """ out: Dict[str, Any] = {} cfg = user_config if isinstance(user_config, dict) else {} for section, key in cls._CACHE_BUSTING_CONFIG_KEYS: section_val = cfg.get(section) if section == "checkpoints" and isinstance(section_val, bool): # Preserve legacy ``checkpoints: true`` behavior. A live # toggle must still rebuild the cached agent. out[f"{section}.{key}"] = section_val if key == "enabled" else None elif isinstance(section_val, dict): out[f"{section}.{key}"] = section_val.get(key) else: out[f"{section}.{key}"] = None try: from tools.registry import registry out["tools.registry_generation"] = getattr(registry, "_generation", None) except Exception: out["tools.registry_generation"] = None # Honcho identity-mapping keys live in honcho.json, not user_config. # Only read that file when Honcho is the active memory provider. provider = cfg_get(cfg, "memory", "provider") if isinstance(provider, str) and provider.lower() == "honcho": out.update(cls._extract_honcho_cache_busting_config()) else: out.update(cls._empty_honcho_cache_busting_config()) return out @staticmethod def _agent_config_signature( model: str, runtime: dict, enabled_toolsets: list, ephemeral_prompt: str, cache_keys: dict | None = None, user_id: str | None = None, user_id_alt: str | None = None, skip_context_files: bool = False, ) -> str: """Compute a stable string key from agent config values. Signature change → cached AIAgent rebuilt; unchanged → reused (frozen prompt + schemas for cache hits). Callers pass ``_extract_cache_busting_config(user_config)`` so config.yaml edits apply on the next message. ``user_id`` / ``user_id_alt`` participate because Honcho freezes them into ``HonchoSessionManager`` at init; omitting them in shared-thread keys (``thread_sessions_per_user=False``) would attribute one user's messages to another's peer. """ import hashlib, json as _j # Fingerprint the FULL credential, not a short prefix: OAuth/JWT-style tokens often share a # common prefix (e.g. "eyJhbGci"), so a prefix would give false cache hits across auth switches. _api_key = str(runtime.get("api_key", "") or "") _api_key_fingerprint = hashlib.sha256(_api_key.encode()).hexdigest() if _api_key else "" _cache_keys_sorted = sorted((cache_keys or {}).items()) blob = _j.dumps( [ model, _api_key_fingerprint, runtime.get("base_url", ""), runtime.get("provider", ""), runtime.get("requested_provider", ""), runtime.get("api_mode", ""), sorted((runtime.get("capabilities") or {}).items()), sorted(enabled_toolsets) if enabled_toolsets else [], # reasoning_config excluded — it's set per-message on the # cached agent and doesn't affect system prompt or tools. ephemeral_prompt or "", _cache_keys_sorted, str(user_id or ""), str(user_id_alt or ""), # skip_context_files changes the agent's frozen system prompt (context files in vs out): # a toggled edit must rebuild the cached agent, not silently reuse it. bool(skip_context_files), ], sort_keys=True, default=str, ) return hashlib.sha256(blob.encode()).hexdigest()[:16] def _rehydrate_session_model_override(self, session_key: str) -> None: """Lazily restore a persisted /model override after a gateway restart. ``_session_model_overrides`` is in-memory only. Non-secret parts (model/provider/base_url) are written through on /model (cleared on /new) and read back here on first use; api_key is never persisted and is re-resolved. No-op when an in-memory override or nothing exists. """ from gateway.run import _resolve_runtime_agent_kwargs_for_provider _rehydrate_state = self._peek_session_state(session_key) if ( _rehydrate_state is not None and _rehydrate_state.conversation.model_override is not None ): return store = getattr(self, "session_store", None) if store is None: return try: persisted = store.get_model_override(session_key) except Exception: logger.debug( "Failed to read persisted session model override", exc_info=True ) return if not persisted: return override: Dict[str, Any] = { "model": persisted.get("model"), "provider": persisted.get("provider"), "base_url": persisted.get("base_url"), } provider = persisted.get("provider") if provider: # Re-resolve credentials for the persisted provider. On failure (e.g. credentials # removed since the switch) keep the credential-less override — # _resolve_session_agent_runtime falls back to env resolution and layers model/provider. try: runtime = _resolve_runtime_agent_kwargs_for_provider(provider) override["api_key"] = runtime.get("api_key") override["api_mode"] = runtime.get("api_mode") override["credential_pool"] = runtime.get("credential_pool") override["request_overrides"] = dict( runtime.get("request_overrides") or {} ) override["requested_provider"] = runtime.get("requested_provider") override["capabilities"] = dict(runtime.get("capabilities") or {}) override["max_tokens"] = runtime.get("max_tokens") if not override.get("base_url"): override["base_url"] = runtime.get("base_url") except Exception: logger.debug( "Credential re-resolution failed for persisted override " "(provider=%s); using credential-less override", provider, exc_info=True, ) self._session_state(session_key).conversation.model_override = override logger.info( "Rehydrated persisted /model override for session=%s: model=%s provider=%s", session_key, override.get("model"), provider or "", ) def _apply_session_model_override( self, session_key: str, model: str, runtime_kwargs: dict ) -> tuple: """Apply /model session overrides if present, returning (model, runtime_kwargs). Overrides take precedence over config.yaml defaults so the switched model is actually used; ``None`` fields are skipped so partial overrides don't clobber valid defaults. """ from gateway.run import _credential_pool_for_provider _apply_state = self._peek_session_state(session_key) override = _apply_state.conversation.model_override if _apply_state else None if not override: return model, runtime_kwargs model = override.get("model", model) for key in ( "provider", "requested_provider", "api_key", "base_url", "api_mode", "credential_pool", "capabilities", "max_tokens", ): val = override.get(key) if val is not None: runtime_kwargs[key] = val # request_overrides reflects the switched-to provider; apply whenever the override recorded # it (even as None) so switching to a provider without configured overrides clears a stale # value left by the default provider's runtime resolution. if "request_overrides" in override: override_request_overrides = override.get("request_overrides") if isinstance(override_request_overrides, dict) and override_request_overrides: runtime_kwargs["request_overrides"] = dict(override_request_overrides) else: runtime_kwargs["request_overrides"] = override_request_overrides if ( runtime_kwargs.get("api_key") and runtime_kwargs.get("credential_pool") is None and override.get("provider") ): runtime_kwargs["credential_pool"] = _credential_pool_for_provider( override.get("provider") ) return model, runtime_kwargs def _snapshot_session_model_override(self, session_key: str) -> dict: """Capture a gateway session override before a one-turn switch.""" _snap_state = self._peek_session_state(session_key) override = _snap_state.conversation.model_override if _snap_state else None return { "had_override": override is not None, "override": dict(override) if override is not None else None, } def _restore_session_model_override(self, session_key: str, snapshot: dict) -> None: """Restore the session override captured before a one-turn switch.""" if not session_key: return if snapshot.get("had_override"): self._session_state(session_key).conversation.model_override = dict( snapshot.get("override") or {} ) else: _rst_state = self._peek_session_state(session_key) if _rst_state is not None: _rst_state.conversation.model_override = None self._evict_cached_agent(session_key) def _is_intentional_model_switch(self, session_key: str, agent_model: str) -> bool: """Return True if *agent_model* matches an active /model session override.""" _ims_state = self._peek_session_state(session_key) override = _ims_state.conversation.model_override if _ims_state else None return override is not None and override.get("model") == agent_model def _release_running_agent_state( self, session_key: str, *, run_generation: Optional[int] = None, ) -> bool: """Pop ALL per-running-agent state entries for ``session_key``; True when cleared. Call at every site that ends a running turn, whatever the cause. State that PERSISTS across turns (model overrides, voice mode, pending approvals, update prompt) is NOT touched. With ``run_generation``, only clear if that generation is still current, so a stale async unwind bumped by /stop or /new cannot clobber a newer run (returns False). """ if not session_key: return False if run_generation is not None and not self._is_session_run_current( session_key, run_generation ): return False state = self._peek_session_state(session_key) if state is not None: lease = state.turn.lease if lease is not None: try: lease.release() except Exception: logger.debug( "Failed to release active session slot", exc_info=True ) # One structured reset instead of a drifting pop-list. Turn-lease tokens are deliberately NOT # cleared here — _release_turn_lease owns them. state.turn.clear() # Turn boundary: a running-agent slot was just released; persist the new (lower) in-flight count # so the dashboard readout stays current. Preserves gateway_state (see _persist_active_agents). self._persist_active_agents() return True def _release_turn_lease(self, session_key: str, run_generation: int) -> bool: """Release the turn lease acquired by (``session_key``, ``run_generation``). Token map is keyed by (routing key, run generation), so a stale unwind pops only ITS token and the registry's identity check refuses it if a newer turn holds the lease. Idempotent. """ if not session_key: return False registry = getattr(self, "_turn_leases", None) state = self._peek_session_state(session_key) if state is None or registry is None: return False turn = state.turn if turn.lease_token is None or turn.lease_generation != run_generation: return False token = turn.lease_token turn.lease_token = None turn.lease_generation = None try: return registry.release(token) except Exception: logger.debug("Failed to release turn lease", exc_info=True) return False def _rebind_turn_lease( self, session_key: str, run_generation: int, new_session_id: str ) -> bool: """Follow a mid-turn session_id rotation with the held turn lease. Compression can rotate ``session_entry.session_id`` mid-turn; the flush targets the NEW id, so the serialization boundary must follow or an alias key resolving the new id could start a concurrent turn the lease never sees. Call at every mid-turn reassignment; no-op if no token. """ if not session_key or not new_session_id: return False registry = getattr(self, "_turn_leases", None) state = self._peek_session_state(session_key) if state is None or registry is None: return False turn = state.turn if turn.lease_token is None or turn.lease_generation != run_generation: return False try: return registry.rebind(turn.lease_token, new_session_id) except Exception: logger.debug("Failed to rebind turn lease", exc_info=True) return False def _clear_conversation_scope(self, session_key: str, *, reason: str) -> None: """Clear ALL conversation-scoped per-session state for ``session_key``. THE single conversation-boundary funnel — call this and nothing else at /new, /resume, auto-reset (idle/daily/suspended), expiry finalization and compression-exhausted reset. New conversation-scoped dicts go in _CONVERSATION_SCOPED_STATE so every boundary picks them up (hand-copied pop-lists drifted). Turn-scoped state (_running_agents/_ts, slot leases, turn-lease tokens) is owned by _release_running_agent_state and NOT cleared. Idle agent-cache eviction is NOT a boundary (a resumed turn rebuilds from these). getattr-guarded. """ from gateway.run import _CONVERSATION_SCOPED_STATE if not session_key: return # Structural clear: every conversation-scoped field resets in one # call — no per-attribute pop-list to drift. state = self._peek_session_state(session_key) if state is not None: state.conversation.clear() # Legacy plain-dict stores still in _CONVERSATION_SCOPED_STATE (not yet folded into # SessionState), e.g. _pending_model_notes. SessionState-backed names resolve to MutableMapping # views (not dict), so the isinstance(dict) guard skips them — already handled above. for attr in _CONVERSATION_SCOPED_STATE: store = getattr(self, attr, None) if isinstance(store, dict): store.pop(session_key, None) self._clear_session_boundary_security_state(session_key) logger.debug( "Cleared conversation scope for %s (%s)", session_key, reason ) def _clear_session_boundary_security_state(self, session_key: str) -> None: """Clear per-session control state that must not survive a boundary switch.""" if not session_key: return pending_skills_reload_notes = getattr( self, "_pending_skills_reload_notes", None ) if isinstance(pending_skills_reload_notes, dict): pending_skills_reload_notes.pop(session_key, None) _sec_state = self._peek_session_state(session_key) if _sec_state is not None: _sec_state.persistent.approvals = None _sec_state.persistent.update_prompt_pending = False try: from tools import slash_confirm as _slash_confirm_mod except Exception: _slash_confirm_mod = None if _slash_confirm_mod is not None: try: _slash_confirm_mod.clear(session_key) except Exception as e: logger.debug( "Failed to clear slash-confirm state for session boundary %s: %s", session_key, e, ) try: from tools.approval import clear_session as _clear_approval_session except Exception: return try: _clear_approval_session(session_key) except Exception as e: logger.debug( "Failed to clear approval state for session boundary %s: %s", session_key, e, ) def _begin_session_run_generation(self, session_key: str) -> int: """Claim a fresh, monotonically increasing run generation token for ``session_key``. If /stop or /new invalidates the token while the old worker is still unwinding, the late result is recognized and dropped instead of bleeding into the fresh session. """ if not session_key: return 0 persistent = self._session_state(session_key).persistent # Monotonic by design (#28686): incremented here, NEVER reset. persistent.run_generation = int(persistent.run_generation) + 1 return persistent.run_generation def _invalidate_session_run_generation(self, session_key: str, *, reason: str = "") -> int: """Invalidate any in-flight run token for ``session_key``.""" generation = self._begin_session_run_generation(session_key) if reason: logger.info( "Invalidated run generation for %s → %d (%s)", session_key, generation, reason, ) return generation def _is_session_run_current(self, session_key: str, generation: int) -> bool: """Return True when ``generation`` is still current for ``session_key``.""" if not session_key: return True state = self._peek_session_state(session_key) current = state.persistent.run_generation if state is not None else 0 return int(current) == int(generation) def _bind_adapter_run_generation( self, adapter: Any, session_key: str, generation: int | None, ) -> None: """Bind a gateway run generation to the adapter's active-session event.""" if not adapter or not session_key or generation is None: return try: interrupt_event = getattr(adapter, "_active_sessions", {}).get(session_key) if interrupt_event is not None: setattr(interrupt_event, "_hermes_run_generation", int(generation)) except Exception: pass async def _interrupt_and_clear_session( self, session_key: str, source: SessionSource, *, interrupt_reason: str, invalidation_reason: str, release_running_state: bool = True, ) -> None: """Interrupt the current run and clear queued session state consistently.""" from gateway.run import _AGENT_PENDING_SENTINEL, _reap_gateway_turn_processes, request_hard_interrupt if not session_key: return _iac_state = self._peek_session_state(session_key) running_agent = _iac_state.turn.agent if _iac_state else None _process_task_id = "" _process_baseline = None if running_agent and running_agent is not _AGENT_PENDING_SENTINEL: request_hard_interrupt(running_agent, interrupt_reason) _process_task_id = getattr( running_agent, "_gateway_turn_process_task_id", "" ) _process_baseline = getattr( running_agent, "_gateway_turn_process_baseline", None ) # Bump the generation BEFORE scheduling the reap thread and capture the post-bump value: # task_id is session-scoped, so a replacement turn spawning before the reap runs bumps it # again and the closure sees a stale generation and skips — the replacement's own baseline # covers its cleanup, so nothing stays unreaped. _generation_at_interrupt = self._invalidate_session_run_generation( session_key, reason=invalidation_reason ) if _process_task_id and _process_baseline is not None: threading.Thread( target=_reap_gateway_turn_processes, args=(_process_task_id, _process_baseline), kwargs={ "source": "gateway_turn_interrupt", "is_still_current": lambda: self._is_session_run_current( session_key, _generation_at_interrupt ), }, name=f"gateway-turn-reaper-{_process_task_id[:12]}", daemon=True, ).start() adapter = self._adapter_for_source(source) interrupt_session_activity = getattr( type(adapter), "interrupt_session_activity", None ) if adapter and callable(interrupt_session_activity): metadata = self._thread_metadata_for_source(source) try: params = inspect.signature(interrupt_session_activity).parameters accepts_metadata = "metadata" in params or any( param.kind is inspect.Parameter.VAR_KEYWORD for param in params.values() ) except (TypeError, ValueError): accepts_metadata = False if accepts_metadata: await adapter.interrupt_session_activity( session_key, source.chat_id, metadata=metadata ) else: await adapter.interrupt_session_activity(session_key, source.chat_id) if adapter and hasattr(adapter, "get_pending_message"): adapter.get_pending_message(session_key) # consume and discard if _iac_state is not None: _iac_state.persistent.pending_command_text = None if release_running_state: self._release_running_agent_state(session_key) # Evict the cached agent: ``_interrupt_requested`` is only cleared by the turn finalizer, # so on a hung/still-draining run the flag survives and silently kills the session's NEXT # message (interrupted=True, api_calls=0, empty response). Like /new and /model, the next # message rebuilds from history; the old agent keeps its flag so a hung drain still dies. self._evict_cached_agent(session_key) async def _refresh_agent_cache_message_count( self, session_key: str, session_id: Optional[str] ) -> None: """Re-baseline a cached agent's stored message_count after THIS turn. The coherence guard compares on-disk ``message_count`` against the BUILD-time snapshot and rebuilds on mismatch; without re-baselining after our own rows flush, every turn would rebuild and destroy prompt caching. Only the count is refreshed (``_sig`` untouched), only if the same agent is still cached, never when the entry records a different ``session_id`` (another conversation's baseline). DB errors leave the snapshot as-is (one spare rebuild). """ from gateway.run import _AGENT_PENDING_SENTINEL if self._session_db is None or not session_id: return _cache_lock = getattr(self, "_agent_cache_lock", None) _cache = getattr(self, "_agent_cache", None) if not _cache_lock or _cache is None: return try: _sess_row = await self._session_db.get_session(session_id) _live = _sess_row.get("message_count", 0) if _sess_row else None except Exception: return if _live is None: return with _cache_lock: cached = _cache.get(session_key) # Only re-baseline a live 3-tuple entry; skip pending sentinels, legacy 2-tuples (they opt # out of the guard), and entries evicted/rebuilt mid-turn. if ( isinstance(cached, tuple) and len(cached) > 2 and cached[0] is not _AGENT_PENDING_SENTINEL ): # A snapshot taken for a different session_id (same session_key, different conversation) # belongs to a different DB row — leave it alone. _snapshot_sid = cached[3] if len(cached) > 3 else None if _snapshot_sid is not None and _snapshot_sid != session_id: return if cached[2] != _live: if _snapshot_sid is None: # Legacy 3-tuple: preserve the 3-element shape for callers indexing ``cached[2]``. _cache[session_key] = (cached[0], cached[1], _live) else: _cache[session_key] = ( cached[0], cached[1], _live, _snapshot_sid, ) def _set_pending_turn_sidecar_notes(self, session_key: str, notes: List[str]) -> None: """Stage per-turn must-deliver notes for the next agent run (one-shot).""" if not session_key or not notes: return self._session_state(session_key).conversation.sidecar_notes = list(notes) def _consume_pending_turn_sidecar_notes(self, session_key: str) -> List[str]: if not session_key: return [] state = self._peek_session_state(session_key) if state is None: return [] staged = state.conversation.sidecar_notes state.conversation.sidecar_notes = [] return list(staged) if isinstance(staged, list) else [] def _voice_channel_sidecar_note(self, event, source: SessionSource, session_key: str) -> Optional[str]: """Return a ``[Voice channel now: ...]`` note when VC state changed. Unchanged state returns ``None`` so per-turn member/speaking churn can't touch the prompt. """ if source.platform != Platform.DISCORD: return None adapter = self.adapters.get(Platform.DISCORD) guild_id = self._get_guild_id(event) if not (guild_id and adapter and hasattr(adapter, "get_voice_channel_context")): return None try: vc_now = adapter.get_voice_channel_context(guild_id) or "" except Exception: logger.debug("voice-channel context read failed", exc_info=True) return None vc_prev = None if session_key: _vc_state = self._session_state(session_key) vc_prev = _vc_state.conversation.vc_last _vc_state.conversation.vc_last = vc_now if vc_now == (vc_prev if vc_prev is not None else ""): return None if not vc_now: return "[Voice channel now: not connected to a voice channel]" return f"[Voice channel now: {vc_now}]" def _pinned_session_context_prompt( self, context, redact_pii: bool, session_key: Optional[str] ) -> str: """Return the session-context prompt, pinned per session. Key hit → pinned bytes reused VERBATIM (immune to renderer nondeterminism); key miss → re-render ``build_session_context_prompt`` and re-pin (rename, topic edit, /sethome, ...). """ _eph_key = self._ephemeral_change_key(context, redact_pii) _eph_pin = None if session_key: _pin_state = self._peek_session_state(session_key) _eph_pin = _pin_state.conversation.ephemeral_pin if _pin_state else None if _eph_pin is not None and _eph_pin[0] == _eph_key: return _eph_pin[1] text = build_session_context_prompt(context, redact_pii=redact_pii) if session_key: self._session_state(session_key).conversation.ephemeral_pin = ( _eph_key, text, ) return text @staticmethod def _ephemeral_change_key(context, redact_pii: bool) -> str: """Hash the exact inputs ``build_session_context_prompt`` renders. Invariant (tests/gateway/test_prompt_tail_freeze.py): any input whose change alters the rendered bytes MUST appear here — omission means a stale pinned prompt; extras only re-render. """ import hashlib src = context.source platform = src.platform.value if src.platform else "" discord_ids: tuple = () discord_tools = "" if src.platform == Platform.DISCORD: from gateway.session import _discord_tools_loaded discord_tools = "1" if _discord_tools_loaded() else "0" discord_ids = ( str(src.guild_id or ""), str(src.parent_chat_id or ""), str(src.thread_id or ""), str(src.chat_id or ""), # Only PRESENCE is rendered (the id itself arrives per-turn in the user message) — # keying on the value would re-render every message for zero byte change. "1" if src.message_id else "0", ) # Slack's capability-aware platform note is gated on _slack_tools_loaded() — the gate state must # be in the key (same parity contract as the Discord gate above) so a config / MCP-registration # flip re-renders once instead of serving a stale pinned note for the rest of the session. slack_tools = "" if src.platform == Platform.SLACK: from gateway.session import _slack_tools_loaded slack_tools = "1" if _slack_tools_loaded() else "0" try: from hermes_constants import display_hermes_home home_display = str(display_hermes_home()) except Exception: home_display = "" key_tuple = ( platform, str(src.chat_id or ""), str(src.thread_id or ""), str(src.chat_type or ""), str(src.chat_name or ""), str(src.chat_topic or ""), str(src.user_name or ""), str(src.user_id or ""), str(getattr(src, "profile", None) or ""), bool(context.shared_multi_user_session), discord_ids, discord_tools, slack_tools, tuple(p.value for p in context.connected_platforms), tuple( ( p.value, str(getattr(hc, "name", "") or ""), str(getattr(hc, "chat_id", "") or ""), ) for p, hc in context.home_channels.items() ), bool(redact_pii), home_display, ) return hashlib.sha256(repr(key_tuple).encode("utf-8")).hexdigest() def _evict_cached_agent(self, session_key: str) -> None: """Remove a cached agent for a session (called on /new, /model, etc). Also soft-releases the evicted agent's LLM client pool (``release_clients()``): AIAgent holds reference cycles that delay collection, so without it gateway RSS grows across /new. Soft = frees clients and per-turn child subagents but PRESERVES the session's terminal sandbox, browser daemon and bg processes (keyed on task_id) since the session may resume. True boundaries (/new) call ``_cleanup_agent_resources`` first (release is idempotent). Cleanup runs on a daemon thread so ``_agent_cache_lock`` never spans slow socket teardown. """ from gateway.run import _AGENT_PENDING_SENTINEL # Prompt-stability state rides the agent-cache lifecycle: a fresh agent must re-render its # session-context bytes (the pin) and re-see the current voice-channel state once. _evict_state = self._peek_session_state(session_key) if _evict_state is not None: _evict_state.conversation.ephemeral_pin = None _evict_state.conversation.vc_last = None _lock = getattr(self, "_agent_cache_lock", None) evicted = None if _lock: with _lock: evicted = self._agent_cache.pop(session_key, None) else: _cache = getattr(self, "_agent_cache", None) if _cache is not None: evicted = _cache.pop(session_key, None) agent = evicted[0] if isinstance(evicted, tuple) and evicted else evicted if agent is None or agent is _AGENT_PENDING_SENTINEL: return # Don't tear down an agent that's actively mid-turn — its client, # sandbox and child subagents are in use by the running request. running_ids = self._running_agent_ids() if id(agent) in running_ids: return try: threading.Thread( target=self._release_evicted_agent_soft, args=(agent,), daemon=True, name=f"agent-evict-{str(session_key)[:24]}", ).start() except Exception: # If we can't spawn a thread (interpreter shutdown), release # inline as a best-effort fallback. with suppress(Exception): self._release_evicted_agent_soft(agent) def _commit_memory_before_soft_evict(self, agent: Any, key: str) -> None: """Fire on_session_end extraction before soft-evicting a live agent. Soft eviction keeps the session resumable and does NOT fire ``on_session_end`` — that is ``_session_expiry_watcher``'s job at true expiry. But the watcher tears down whatever it finds in ``_agent_cache``; if the LRU cap soft-evicts first, memory providers never see the transcript. So commit extraction here via ``commit_memory_session`` (no teardown). Only for finalizable sessions — ``mode == "none"`` never finalizes. Best-effort: failures swallowed. """ if agent is None or not hasattr(agent, "commit_memory_session"): return if getattr(agent, "_memory_manager", None) is None: return # no external memory provider — nothing to commit try: _store = getattr(self, "session_store", None) if _store is None: return _store._ensure_loaded() entry = _store._entries.get(key) if entry is None: return # Compensate only when the watcher would expect this agent at expiry (finite policy, not yet # expired). Expired sessions are torn down by the watcher; mode="none" is never finalized. if not _store.is_session_finalizable(entry): return if _store._is_session_expired(entry): return messages = getattr(agent, "_session_messages", None) agent.commit_memory_session(messages if isinstance(messages, list) else None) logger.debug( "Committed on_session_end extraction before soft-evicting " "finalizable session=%s (cache pressure, pre-expiry)", key, ) except Exception as _e: logger.debug("Pre-evict memory commit failed for %s: %s", key, _e) def _commit_then_release_soft(self, agent: Any, key: str) -> None: """Commit end-of-session memory (if warranted), then soft-release. Runs on the daemon eviction thread so neither blocks the caller's held cache lock. Order matters: commit needs the live memory manager before ``release_clients`` drops the buffer. """ self._commit_memory_before_soft_evict(agent, key) self._release_evicted_agent_soft(agent) def _release_evicted_agent_soft(self, agent: Any) -> None: """Soft cleanup for cache-evicted agents — preserves session tool state. Unlike _cleanup_agent_resources (full teardown), an evicted session may resume, so its terminal sandbox, browser daemon and bg processes must outlive the AIAgent instance. """ if agent is None: return try: if hasattr(agent, "release_clients"): agent.release_clients() else: # Older agent instance (shouldn't happen in practice) — # fall back to the legacy full-close path. self._cleanup_agent_resources(agent) except Exception: pass # Free conversation history — tens of MB of tool output on heavy 100+-tool-call sessions. # release_clients() preserves session tool state for resume, but the message list is rebuilt from # persisted session JSON on the next turn, so dropping it here is safe. if hasattr(agent, "_session_messages"): agent._session_messages = [] # _db_flush_scan_prefix (run_agent.py, stamped on every successful flush) is a shallow copy # sharing every message dict of the flushed transcript, so leaving it pins the multi-MB strings # this eviction frees. Pressure-evictable agents have flushed by definition, so it's populated. if hasattr(agent, "_db_flush_scan_prefix"): agent._db_flush_scan_prefix = None def _agent_cache_bounds(self): """Operator-configured agent-cache bounds, resolved once per process. Resolved lazily rather than in ``__init__`` so it also works for the ``__new__``-constructed runners used by tests and by the slash-command mixin. """ from gateway.run import _load_gateway_config bounds = getattr(self, "_agent_cache_bounds_cache", None) if bounds is None: from gateway.agent_cache_pressure import resolve_agent_cache_bounds try: bounds = resolve_agent_cache_bounds(_load_gateway_config()) except Exception as _e: logger.debug("Agent cache bounds config read failed: %s", _e) # Resolve from an empty config rather than bare AgentCacheBounds(): the dataclass default # has memory_high_mb=None (pressure pass OFF) but an *absent* section means "auto" — a # transient config read failure must not permanently disable the OOM valve. bounds = resolve_agent_cache_bounds({}) self._agent_cache_bounds_cache = bounds return bounds def _agent_cache_cap(self) -> int: """Effective LRU cap — the configured override, else the default.""" from gateway.run import _AGENT_CACHE_MAX_SIZE configured = self._agent_cache_bounds().max_size return configured if configured else _AGENT_CACHE_MAX_SIZE def _agent_cache_idle_ttl(self) -> float: """Effective idle TTL in seconds — configured override, else default.""" from gateway.run import _AGENT_CACHE_IDLE_TTL_SECS configured = self._agent_cache_bounds().idle_ttl_secs return configured if configured else _AGENT_CACHE_IDLE_TTL_SECS def _sweep_agent_cache_under_pressure(self) -> int: """Shed cached transcripts once the gateway heap nears its budget; returns count evicted. The LRU cap counts entries and the idle sweep counts seconds; neither knows one cached agent pins a full ``_session_messages`` transcript (tens of MB). Warm and finalizable agents are never swept, so RSS climbs until the cgroup throttles. Above the anonymous-RSS budget this soft-evicts LRU agents (transcript rebuilt from the persisted session next turn). Never touched: agents mid-turn, the most recently used sessions, and transcripts not yet on disk. """ from gateway.run import _AGENT_PENDING_SENTINEL from gateway.agent_cache_pressure import ( plan_pressure_evictions, read_anon_rss_mb, transcript_persistence_caught_up, ) bounds = self._agent_cache_bounds() if not bounds.memory_high_mb: return 0 _cache = getattr(self, "_agent_cache", None) _lock = getattr(self, "_agent_cache_lock", None) if not _cache or _lock is None: # Nothing cached — whatever is using the heap, it isn't us, and # warning about it every tick would point at the wrong subsystem. return 0 rss_mb = read_anon_rss_mb() if rss_mb is None or rss_mb < bounds.memory_high_mb: return 0 running_ids = self._running_agent_ids() def _is_evictable(key: str, agent: Any) -> bool: if agent is None or agent is _AGENT_PENDING_SENTINEL: return False if id(agent) in running_ids: return False return transcript_persistence_caught_up(agent) with _lock: ordered = [ (key, entry[0] if isinstance(entry, tuple) and entry else entry) for key, entry in _cache.items() ] plan = plan_pressure_evictions( ordered, is_evictable=_is_evictable, max_evictions=bounds.max_evictions_per_pass, protect_recent=bounds.protect_recent, ) for key, _ in plan: _cache.pop(key, None) if not plan: _mid_turn = sum(1 for _, a in ordered if a is not None and id(a) in running_ids) _unflushed = sum( 1 for _, a in ordered if a is not None and a is not _AGENT_PENDING_SENTINEL and id(a) not in running_ids and not transcript_persistence_caught_up(a) ) logger.warning( "Agent cache pressure: anon RSS %dMB over budget %dMB but no " "evictable session (%d cached, %d mid-turn, %d blocked on " "un-flushed persistence)%s", rss_mb, bounds.memory_high_mb, len(ordered), _mid_turn, _unflushed, ( " — transcripts are not reaching the session DB " "(session persistence disabled or failing?); the memory " "valve cannot shed sessions until they persist." if _unflushed and not _mid_turn else " — memory will keep climbing until those turns finish." ), ) return 0 evicted_count = len(plan) logger.warning( "Agent cache pressure: anon RSS %dMB over budget %dMB — evicting " "%d LRU session(s): %s", rss_mb, bounds.memory_high_mb, evicted_count, ", ".join(key for key, _ in plan), ) try: threading.Thread( target=self._release_pressure_batch, args=(plan,), daemon=True, name="agent-cache-pressure", ).start() except Exception: self._release_pressure_batch(plan) # NOTE: _release_pressure_batch drains `plan` in place (so the trim runs with no lingering # agent refs) — len(plan) is 0 once the daemon thread finishes, hence the pre-captured count. return evicted_count def _release_pressure_batch(self, plan: List[tuple]) -> None: """Release a pressure-evicted batch, then return the heap to the OS. Sequential on one daemon thread (the batch is capped; the goal is reclaiming memory, not racing teardowns). The trailing ``malloc_trim`` makes RSS actually fall — glibc otherwise keeps freed arenas. The plan is drained (``pop`` + ``del``), not iterated, so no local reference pins evicted agents during ``gc.collect`` + trim (else the valve over-evicts). """ while plan: key, agent = plan.pop(0) # FIFO — evict LRU-first order preserved try: self._commit_then_release_soft(agent, key) except Exception as _e: logger.debug("Pressure release failed for %s: %s", key, _e) del agent try: from hermes_cli.mem_trim import trim_memory trim_memory(force=True, reason="agent_cache_pressure") except Exception: pass def _enforce_agent_cache_cap(self) -> None: """Evict oldest cached agents when cache exceeds the LRU cap. Requires _agent_cache_lock. Resource cleanup runs on a daemon thread so the lock is not held over slow teardown. Agents in _running_agents are SKIPPED (their clients/sandboxes/subagents are in use); if every LRU candidate is active the cache stays over cap until the next insert. """ _cache = getattr(self, "_agent_cache", None) if _cache is None: return # OrderedDict.popitem(last=False) pops oldest; plain dict lacks the # arg so skip enforcement if a test fixture swapped the cache type. if not hasattr(_cache, "move_to_end"): return # Snapshot of agent instances mid-turn, keyed by id() so lookup is O(1) and independent of # AIAgent.__eq__ (which MagicMock overrides in tests). running_ids = self._running_agent_ids() # Walk LRU → MRU; only the first (size - cap) LRU positions are candidates. An active slot is # SKIPPED rather than evicting a newer entry — that would penalise a fresh session (no cache # history) to protect a long-running one. Cache may stay over cap until the next insert. cap = self._agent_cache_cap() excess = max(0, len(_cache) - cap) evict_plan: List[tuple] = [] # [(key, agent), ...] if excess > 0: ordered_keys = list(_cache.keys()) for key in ordered_keys[:excess]: entry = _cache.get(key) agent = entry[0] if isinstance(entry, tuple) and entry else None if agent is not None and id(agent) in running_ids: continue # active mid-turn; don't evict, don't substitute evict_plan.append((key, agent)) for key, _ in evict_plan: _cache.pop(key, None) remaining_over_cap = len(_cache) - cap if remaining_over_cap > 0: logger.warning( "Agent cache over cap (%d > %d); %d excess slot(s) held by " "mid-turn agents — will re-check on next insert.", len(_cache), cap, remaining_over_cap, ) for key, agent in evict_plan: logger.info( "Agent cache at cap; evicting LRU session=%s (cache_size=%d)", key, len(_cache), ) if agent is not None: # Commit end-of-session memory, then soft-release, both on the daemon thread so the # (possibly network-bound) provider call never blocks the held cache lock. threading.Thread( target=self._commit_then_release_soft, args=(agent, key), daemon=True, name=f"agent-cache-evict-{key[:24]}", ).start() def _sweep_idle_cached_agents(self) -> int: """Evict cached agents idle past the idle TTL; returns the number evicted. Acquires the cache lock internally (safe from the expiry watcher); cleanup on daemon threads. Agents in _running_agents are SKIPPED — tearing down an active turn crashes it. """ _cache = getattr(self, "_agent_cache", None) _lock = getattr(self, "_agent_cache_lock", None) if _cache is None or _lock is None: return 0 now = time.time() idle_ttl = self._agent_cache_idle_ttl() to_evict: List[tuple] = [] running_ids = self._running_agent_ids() with _lock: for key, entry in list(_cache.items()): agent = entry[0] if isinstance(entry, tuple) and entry else None if agent is None: continue if id(agent) in running_ids: continue # mid-turn — don't tear it down last_activity = getattr(agent, "_last_activity_ts", None) if last_activity is None: continue if (now - last_activity) > idle_ttl: # If the session hasn't actually expired in the store (e.g. daily-reset fires hours # after the last message), keep the agent cached so the expiry watcher can still find # it and call on_session_end() with the live transcript. BUT only defer when the # watcher will EVER finalize it: for mode == "none" (is_session_finalizable() False) # deferring pins the agent for the gateway's lifetime — the leak this sweep relieves. # Those fall through to soft eviction WITHOUT on_session_end, correctly (never a # session-end boundary). Finite sessions evicted under LRU-cap pressure are covered # by _commit_memory_before_soft_evict on the cap path. session_entry = None _store = getattr(self, "session_store", None) try: if _store is not None: _store._ensure_loaded() session_entry = _store._entries.get(key) except Exception: session_entry = None if ( session_entry is not None and _store is not None and _store.is_session_finalizable(session_entry) and not _store._is_session_expired(session_entry) ): continue # keep agent — finite session hasn't expired to_evict.append((key, agent)) for key, _ in to_evict: _cache.pop(key, None) for key, agent in to_evict: logger.info( "Agent cache idle-TTL evict: session=%s (idle=%.0fs)", key, now - getattr(agent, "_last_activity_ts", now), ) threading.Thread( target=self._release_evicted_agent_soft, args=(agent,), daemon=True, name=f"agent-cache-idle-{key[:24]}", ).start() return len(to_evict)