"""Durable cross-process session turn lease for ``TurnFacadeMixin.run_conversation``. One process at a time may load -> run -> flush a session shared through state.db (Desktop, CLI resume, gateway, background delivery). ``admit_durable_turn_lease`` acquires the row lease (or returns the early result the façade must hand back); ``DurableTurnLease`` owns the refresher daemon thread, the turn-liveness watchdog wiring, and the lease-loss / stall interrupt plumbing. """ import logging import os import threading from dataclasses import dataclass from typing import Any, Dict, List, Optional # Same logger name as the origin module so log records / caplog filters are unchanged. logger = logging.getLogger("run_agent") LEASE_TTL_SECONDS = 300.0 LEASE_WAIT_SECONDS = 1800.0 class DurableTurnLease: """An admitted session turn lease plus the threads that keep it alive and watch the turn. ``stop`` is shared by the refresher and the liveness watchdog; ``turn_active`` gates every interrupt so a late refresher miss can never hard-interrupt the NEXT turn. Both are read and written only under ``_lock``. """ def __init__(self, agent, db, session_id: str, holder: str) -> None: self.agent = agent self.db = db self.session_id = session_id # id at admission; release always targets this row self.holder = holder self.stop = threading.Event() self.refresh_interval = float(getattr(agent, "_session_turn_lease_refresh_interval", 60.0)) self._lock = threading.Lock() self.turn_active = False self.interrupt_message: Optional[str] = None self.refresh_thread: Optional[threading.Thread] = None self.liveness_thread: Optional[threading.Thread] = None def _current_session_id(self) -> str: return getattr(self.agent, "session_id", None) or self.session_id def build_threads(self) -> None: """Create (not start) the refresher thread and, when configured, the liveness watchdog. Lease renewal is NOT evidence of progress; a silently stalled turn would renew forever, so the watchdog (policy in ``agent/turn_liveness.py``) is wired with the commit/deactivate callbacks below. """ self.refresh_thread = threading.Thread( target=self.refresh_loop, name="session-turn-lease-refresh", daemon=True ) try: from hermes_cli.config import load_config_readonly liveness_config = load_config_readonly() or {} except Exception: liveness_config = {} from agent import turn_liveness timeout_s, poll_s = turn_liveness.resolve_turn_liveness_settings(liveness_config) if timeout_s is not None: self.liveness_thread = turn_liveness.TurnLivenessWatchdog( self.agent, session_id=self._current_session_id(), timeout_s=timeout_s, poll_s=poll_s, stop_event=self.stop, activity_lock=self.agent._liveness_activity_lock(), is_turn_active=self.is_turn_active, commit_abort=self.commit_liveness_abort, deactivate_turn=self.deactivate_after_liveness_abort, ).make_thread() def start(self) -> None: with self._lock: self.turn_active = True # Stamp the activity clock at turn entry: `_last_activity_ts` persists across turns, so without # this the watchdog would measure idle from the PREVIOUS turn and abort a fresh one. self.agent._touch_activity("starting new turn") self.refresh_thread.start() if self.liveness_thread is not None: self.liveness_thread.start() def stop_refresher(self) -> None: with self._lock: self.turn_active = False self.stop.set() def join_threads(self, timeout: float = 1.0) -> None: for thread in (self.refresh_thread, self.liveness_thread): if thread is not None and thread.is_alive(): thread.join(timeout=timeout) def release(self) -> None: """Release the row and drop the agent's holder attrs (only if they still name this lease).""" agent = self.agent try: self.db.release_session_turn_lease(self.session_id, self.holder) except Exception: logger.error("Failed to release session turn lease: %s", self.session_id, exc_info=True) if getattr(agent, "_active_session_turn_lease_holder", None) == self.holder: agent._active_session_turn_lease_holder = None agent._active_session_turn_lease_ttl_seconds = None def is_turn_active(self) -> bool: with self._lock: return self.turn_active def _interrupt_turn(self, message: str) -> None: # Lease-loss interrupts fire UNCONDITIONALLY (no generation claim): a lost lease means this # process no longer owns the session. Only the watchdog's stalls can be spuriously stale. agent = self.agent with self._lock: if self.stop.is_set() or not self.turn_active: return self.interrupt_message = message try: agent.interrupt(message, hard_cancel=True) except Exception: agent._interrupt_requested = True agent._interrupt_message = message def commit_liveness_abort(self, snapshot, message: str) -> bool: """Commit point for the watchdog's stall observation. Revalidates the observed ``(generation, timestamp)`` under the SAME lock ``_touch_activity`` uses, so a turn that resumed while the stall was logged is never hard-cancelled. The revalidated generation is carried into ``interrupt`` as ``require_generation``, which consumes it with the first publication in ONE critical section. If ``interrupt`` raises, the abort declines FAIL-CLOSED. Returns False when stale or already winding down. """ agent = self.agent with agent._liveness_activity_lock(): current_generation = getattr(agent, "_turn_liveness_activity_generation", 0) if (current_generation, getattr(agent, "_last_activity_ts", None)) != ( snapshot.generation, snapshot.activity_ts, ): return False with self._lock: if self.stop.is_set() or not self.turn_active: return False try: published = agent.interrupt( message, hard_cancel=True, require_generation=current_generation ) except Exception: # Fail closed: an exceptional path must not turn an unvalidated claim into abort authority. logger.debug( "Turn liveness abort interrupt raised; declining the abort", exc_info=True ) published = False if published is False: # Claim went stale between revalidation and the hammer: real progress landed. return False with self._lock: self.interrupt_message = message return True def deactivate_after_liveness_abort(self) -> None: """Stop lease renewal after a committed liveness abort. A wedge the hard interrupt cannot unwind must not keep the lease alive forever; TTL expiry lets stale-turn cleanup reclaim the row. """ with self._lock: self.stop.set() self.turn_active = False def clear_interrupt(self) -> None: """Clear only the interrupt admitted by this lease's refresher/watchdog. Run AFTER join.""" message = self.interrupt_message if not message: return agent = self.agent def _clear_if_owned() -> None: # Lazy via the façade so ``patch("agent.turn_facade._set_interrupt")`` keeps intercepting. from agent.turn_facade import _set_interrupt if getattr(agent, "_interrupt_message", None) != message: return agent._interrupt_requested = False agent._interrupt_message = None getattr(agent, "_hard_interrupt_requested", threading.Event()).clear() agent._interrupt_thread_signal_pending = False if agent._execution_thread_id is not None: _set_interrupt(False, agent._execution_thread_id) redirect_lock = getattr(agent, "_pending_redirect_lock", None) if redirect_lock is None: _clear_if_owned() else: with redirect_lock: _clear_if_owned() def refresh_loop(self) -> None: """Renew the lease every ``refresh_interval``; a miss or error interrupts the turn. Long turns outlive the fixed TTL; the holder-qualified UPDATE fences a late refresher from a successor lease. The façade's finally sets ``stop`` before releasing, so a holder-fenced miss observed after stop is not a loss. """ while not self.stop.wait(self.refresh_interval): try: if self.db.refresh_session_turn_lease( self._current_session_id(), self.holder, ttl_seconds=LEASE_TTL_SECONDS ): continue if self.stop.is_set(): return logger.error( "Lost session turn lease while turn is active: %s", self._current_session_id() ) self._interrupt_turn( "Session turn lease lost; stopping to protect the transcript." ) except Exception: if self.stop.is_set(): return logger.warning( "Failed to refresh session turn lease: %s", self._current_session_id(), exc_info=True, ) self._interrupt_turn( "Session turn lease could not be refreshed; stopping to protect the transcript." ) return @dataclass class TurnLeaseAdmission: """Outcome of ``admit_durable_turn_lease``: exactly one of ``lease`` / ``early_result`` may be set.""" lease: Optional[DurableTurnLease] = None early_result: Optional[Dict[str, Any]] = None conversation_history: Optional[List[Dict[str, Any]]] = None def _durable_session_exists(db, session_id: str) -> bool: try: return db.get_session(session_id) is not None except Exception: # A locked / non-WAL read is not proof the row is absent; treating probe failure as "fresh" # ran fail-open at the exact contention point. Acquire, or fail closed. logger.warning( "Could not check durable session before turn lease; " "will acquire rather than run without serialization", exc_info=True, ) return True def admit_durable_turn_lease( agent, *, session_id: str, relay_turn_id: str, task_context: Dict[str, Any], conversation_history: Optional[List[Dict[str, Any]]], ) -> TurnLeaseAdmission: """Acquire the session turn lease when the session is durable; build (not start) its threads. Mutates ``task_context["session_id"]`` and ``agent.session_id`` when the wait forced a resume-id reload. Returns an ``early_result`` (interrupted / timed out) instead of a lease when admission fails; the caller returns it verbatim. """ db = getattr(agent, "_session_db", None) admission = TurnLeaseAdmission(conversation_history=conversation_history) if db is None or not session_id: return admission # A fresh session id has no durable transcript to race over, and callers may supply an in-memory # seed before the row exists — reloading would erase it. Check the concrete type: MagicMock-style # shims accept any attribute without the protocol. exists = _durable_session_exists(db, session_id) if ( getattr(agent, "_persist_disabled", False) or not exists or not callable(getattr(type(db), "acquire_session_turn_lease", None)) ): return admission # Row proven to exist — suppress the redundant create attempt. agent._session_db_created = True holder = ( f"pid={os.getpid()}:turn={relay_turn_id}:platform={task_context['platform'] or 'unknown'}" ) waited = False def _on_wait(elapsed: float) -> None: nonlocal waited waited = True if elapsed < 1.0: agent._emit_status( "⏳ Another Hermes process is using this session; " "waiting for it to finish before starting your turn..." ) else: agent._emit_status( "⏳ Still waiting for the other Hermes process on " f"this session ({int(elapsed)}s)..." ) if not db.acquire_session_turn_lease( session_id, holder, ttl_seconds=LEASE_TTL_SECONDS, wait_seconds=LEASE_WAIT_SECONDS, on_wait=_on_wait, should_abort=lambda: getattr(agent, "_interrupt_requested", False), ): admission.early_result = _lease_not_acquired_result(agent, session_id, conversation_history) return admission # Assign only after admission so the finally cannot release a holder that never owned the row; # persist paths read the agent attr so a late flush is fenced in the same SQLite transaction. lease = DurableTurnLease(agent, db, session_id, holder) agent._active_session_turn_lease_holder = holder agent._active_session_turn_lease_ttl_seconds = LEASE_TTL_SECONDS try: if waited: agent._emit_status("Session is free; loading the latest transcript...") # The holder may have compressed/rotated the session while we waited: reload only AFTER # admission, and skip when acquisition was immediate (avoids a needless prompt-cache miss). latest_session_id = db.resolve_resume_session_id(session_id) if latest_session_id: agent.session_id = latest_session_id task_context["session_id"] = latest_session_id admission.conversation_history = db.get_messages_as_conversation( agent.session_id, repair_alternation=True, include_row_ids=True ) lease.build_threads() except BaseException: # The façade never saw this lease; release here so an admitted row is not leaked. lease.release() raise admission.lease = lease return admission def _lease_not_acquired_result(agent, session_id: str, conversation_history) -> Dict[str, Any]: base = {"messages": list(conversation_history or []), "api_calls": 0, "completed": False} if getattr(agent, "_interrupt_requested", False): logger.info("session turn lease wait aborted by interrupt: %s", session_id) result = { "final_response": ( "Stopped waiting for another Hermes process on this session. " "Your message was not processed." ), **base, "interrupted": True, } interrupt_message = getattr(agent, "_interrupt_message", None) if interrupt_message: result["interrupt_message"] = interrupt_message # The finalizer never runs on this early return; clear so a cached agent doesn't fail-close # the next turn. try: agent.clear_interrupt() except Exception: agent._interrupt_requested = False agent._interrupt_message = None return result # Fail closed like gateway TurnLeaseTimeoutError: surface a resend notice, not a bare TimeoutError. timeout_msg = ( "⏳ Another Hermes process kept this session busy too long. Your message was not " "processed - wait for the other process to finish, then send it again." ) logger.error("session turn lease wait timed out for %s", session_id) try: agent._emit_warning(timeout_msg) except Exception: logger.debug("Failed to emit session turn lease timeout warning", exc_info=True) return { "final_response": timeout_msg, **base, "failed": True, "error": f"session_turn_lease_timeout:{session_id}", }